linux/drivers/xen/events/events_fifo.c
<<
>>
Prefs
   1/*
   2 * Xen event channels (FIFO-based ABI)
   3 *
   4 * Copyright (C) 2013 Citrix Systems R&D ltd.
   5 *
   6 * This source code is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU General Public License as
   8 * published by the Free Software Foundation; either version 2 of the
   9 * License, or (at your option) any later version.
  10 *
  11 * Or, when distributed separately from the Linux kernel or
  12 * incorporated into other software packages, subject to the following
  13 * license:
  14 *
  15 * Permission is hereby granted, free of charge, to any person obtaining a copy
  16 * of this source file (the "Software"), to deal in the Software without
  17 * restriction, including without limitation the rights to use, copy, modify,
  18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19 * and to permit persons to whom the Software is furnished to do so, subject to
  20 * the following conditions:
  21 *
  22 * The above copyright notice and this permission notice shall be included in
  23 * all copies or substantial portions of the Software.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31 * IN THE SOFTWARE.
  32 */
  33
  34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  35
  36#include <linux/linkage.h>
  37#include <linux/interrupt.h>
  38#include <linux/irq.h>
  39#include <linux/smp.h>
  40#include <linux/percpu.h>
  41#include <linux/cpu.h>
  42
  43#include <asm/barrier.h>
  44#include <asm/sync_bitops.h>
  45#include <asm/xen/hypercall.h>
  46#include <asm/xen/hypervisor.h>
  47
  48#include <xen/xen.h>
  49#include <xen/xen-ops.h>
  50#include <xen/events.h>
  51#include <xen/interface/xen.h>
  52#include <xen/interface/event_channel.h>
  53#include <xen/page.h>
  54
  55#include "events_internal.h"
  56
  57#define EVENT_WORDS_PER_PAGE (XEN_PAGE_SIZE / sizeof(event_word_t))
  58#define MAX_EVENT_ARRAY_PAGES (EVTCHN_FIFO_NR_CHANNELS / EVENT_WORDS_PER_PAGE)
  59
  60struct evtchn_fifo_queue {
  61        uint32_t head[EVTCHN_FIFO_MAX_QUEUES];
  62};
  63
  64static DEFINE_PER_CPU(struct evtchn_fifo_control_block *, cpu_control_block);
  65static DEFINE_PER_CPU(struct evtchn_fifo_queue, cpu_queue);
  66static event_word_t *event_array[MAX_EVENT_ARRAY_PAGES] __read_mostly;
  67static unsigned event_array_pages __read_mostly;
  68
  69/*
  70 * sync_set_bit() and friends must be unsigned long aligned.
  71 */
  72#if BITS_PER_LONG > 32
  73
  74#define BM(w) (unsigned long *)((unsigned long)w & ~0x7UL)
  75#define EVTCHN_FIFO_BIT(b, w) \
  76    (((unsigned long)w & 0x4UL) ? (EVTCHN_FIFO_ ##b + 32) : EVTCHN_FIFO_ ##b)
  77
  78#else
  79
  80#define BM(w) ((unsigned long *)(w))
  81#define EVTCHN_FIFO_BIT(b, w) EVTCHN_FIFO_ ##b
  82
  83#endif
  84
  85static inline event_word_t *event_word_from_port(unsigned port)
  86{
  87        unsigned i = port / EVENT_WORDS_PER_PAGE;
  88
  89        return event_array[i] + port % EVENT_WORDS_PER_PAGE;
  90}
  91
  92static unsigned evtchn_fifo_max_channels(void)
  93{
  94        return EVTCHN_FIFO_NR_CHANNELS;
  95}
  96
  97static unsigned evtchn_fifo_nr_channels(void)
  98{
  99        return event_array_pages * EVENT_WORDS_PER_PAGE;
 100}
 101
 102static int init_control_block(int cpu,
 103                              struct evtchn_fifo_control_block *control_block)
 104{
 105        struct evtchn_fifo_queue *q = &per_cpu(cpu_queue, cpu);
 106        struct evtchn_init_control init_control;
 107        unsigned int i;
 108
 109        /* Reset the control block and the local HEADs. */
 110        clear_page(control_block);
 111        for (i = 0; i < EVTCHN_FIFO_MAX_QUEUES; i++)
 112                q->head[i] = 0;
 113
 114        init_control.control_gfn = virt_to_gfn(control_block);
 115        init_control.offset      = 0;
 116        init_control.vcpu        = xen_vcpu_nr(cpu);
 117
 118        return HYPERVISOR_event_channel_op(EVTCHNOP_init_control, &init_control);
 119}
 120
 121static void free_unused_array_pages(void)
 122{
 123        unsigned i;
 124
 125        for (i = event_array_pages; i < MAX_EVENT_ARRAY_PAGES; i++) {
 126                if (!event_array[i])
 127                        break;
 128                free_page((unsigned long)event_array[i]);
 129                event_array[i] = NULL;
 130        }
 131}
 132
 133static void init_array_page(event_word_t *array_page)
 134{
 135        unsigned i;
 136
 137        for (i = 0; i < EVENT_WORDS_PER_PAGE; i++)
 138                array_page[i] = 1 << EVTCHN_FIFO_MASKED;
 139}
 140
 141static int evtchn_fifo_setup(struct irq_info *info)
 142{
 143        unsigned port = info->evtchn;
 144        unsigned new_array_pages;
 145        int ret;
 146
 147        new_array_pages = port / EVENT_WORDS_PER_PAGE + 1;
 148
 149        if (new_array_pages > MAX_EVENT_ARRAY_PAGES)
 150                return -EINVAL;
 151
 152        while (event_array_pages < new_array_pages) {
 153                void *array_page;
 154                struct evtchn_expand_array expand_array;
 155
 156                /* Might already have a page if we've resumed. */
 157                array_page = event_array[event_array_pages];
 158                if (!array_page) {
 159                        array_page = (void *)__get_free_page(GFP_KERNEL);
 160                        if (array_page == NULL) {
 161                                ret = -ENOMEM;
 162                                goto error;
 163                        }
 164                        event_array[event_array_pages] = array_page;
 165                }
 166
 167                /* Mask all events in this page before adding it. */
 168                init_array_page(array_page);
 169
 170                expand_array.array_gfn = virt_to_gfn(array_page);
 171
 172                ret = HYPERVISOR_event_channel_op(EVTCHNOP_expand_array, &expand_array);
 173                if (ret < 0)
 174                        goto error;
 175
 176                event_array_pages++;
 177        }
 178        return 0;
 179
 180  error:
 181        if (event_array_pages == 0)
 182                panic("xen: unable to expand event array with initial page (%d)\n", ret);
 183        else
 184                pr_err("unable to expand event array (%d)\n", ret);
 185        free_unused_array_pages();
 186        return ret;
 187}
 188
 189static void evtchn_fifo_bind_to_cpu(struct irq_info *info, unsigned cpu)
 190{
 191        /* no-op */
 192}
 193
 194static void evtchn_fifo_clear_pending(unsigned port)
 195{
 196        event_word_t *word = event_word_from_port(port);
 197        sync_clear_bit(EVTCHN_FIFO_BIT(PENDING, word), BM(word));
 198}
 199
 200static void evtchn_fifo_set_pending(unsigned port)
 201{
 202        event_word_t *word = event_word_from_port(port);
 203        sync_set_bit(EVTCHN_FIFO_BIT(PENDING, word), BM(word));
 204}
 205
 206static bool evtchn_fifo_is_pending(unsigned port)
 207{
 208        event_word_t *word = event_word_from_port(port);
 209        return sync_test_bit(EVTCHN_FIFO_BIT(PENDING, word), BM(word));
 210}
 211
 212static bool evtchn_fifo_test_and_set_mask(unsigned port)
 213{
 214        event_word_t *word = event_word_from_port(port);
 215        return sync_test_and_set_bit(EVTCHN_FIFO_BIT(MASKED, word), BM(word));
 216}
 217
 218static void evtchn_fifo_mask(unsigned port)
 219{
 220        event_word_t *word = event_word_from_port(port);
 221        sync_set_bit(EVTCHN_FIFO_BIT(MASKED, word), BM(word));
 222}
 223
 224static bool evtchn_fifo_is_masked(unsigned port)
 225{
 226        event_word_t *word = event_word_from_port(port);
 227        return sync_test_bit(EVTCHN_FIFO_BIT(MASKED, word), BM(word));
 228}
 229/*
 230 * Clear MASKED, spinning if BUSY is set.
 231 */
 232static void clear_masked(volatile event_word_t *word)
 233{
 234        event_word_t new, old, w;
 235
 236        w = *word;
 237
 238        do {
 239                old = w & ~(1 << EVTCHN_FIFO_BUSY);
 240                new = old & ~(1 << EVTCHN_FIFO_MASKED);
 241                w = sync_cmpxchg(word, old, new);
 242        } while (w != old);
 243}
 244
 245static void evtchn_fifo_unmask(unsigned port)
 246{
 247        event_word_t *word = event_word_from_port(port);
 248
 249        BUG_ON(!irqs_disabled());
 250
 251        clear_masked(word);
 252        if (evtchn_fifo_is_pending(port)) {
 253                struct evtchn_unmask unmask = { .port = port };
 254                (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
 255        }
 256}
 257
 258static uint32_t clear_linked(volatile event_word_t *word)
 259{
 260        event_word_t new, old, w;
 261
 262        w = *word;
 263
 264        do {
 265                old = w;
 266                new = (w & ~((1 << EVTCHN_FIFO_LINKED)
 267                             | EVTCHN_FIFO_LINK_MASK));
 268        } while ((w = sync_cmpxchg(word, old, new)) != old);
 269
 270        return w & EVTCHN_FIFO_LINK_MASK;
 271}
 272
 273static void handle_irq_for_port(unsigned port)
 274{
 275        int irq;
 276
 277        irq = get_evtchn_to_irq(port);
 278        if (irq != -1)
 279                generic_handle_irq(irq);
 280}
 281
 282static void consume_one_event(unsigned cpu,
 283                              struct evtchn_fifo_control_block *control_block,
 284                              unsigned priority, unsigned long *ready,
 285                              bool drop)
 286{
 287        struct evtchn_fifo_queue *q = &per_cpu(cpu_queue, cpu);
 288        uint32_t head;
 289        unsigned port;
 290        event_word_t *word;
 291
 292        head = q->head[priority];
 293
 294        /*
 295         * Reached the tail last time?  Read the new HEAD from the
 296         * control block.
 297         */
 298        if (head == 0) {
 299                virt_rmb(); /* Ensure word is up-to-date before reading head. */
 300                head = control_block->head[priority];
 301        }
 302
 303        port = head;
 304        word = event_word_from_port(port);
 305        head = clear_linked(word);
 306
 307        /*
 308         * If the link is non-zero, there are more events in the
 309         * queue, otherwise the queue is empty.
 310         *
 311         * If the queue is empty, clear this priority from our local
 312         * copy of the ready word.
 313         */
 314        if (head == 0)
 315                clear_bit(priority, ready);
 316
 317        if (evtchn_fifo_is_pending(port) && !evtchn_fifo_is_masked(port)) {
 318                if (unlikely(drop))
 319                        pr_warn("Dropping pending event for port %u\n", port);
 320                else
 321                        handle_irq_for_port(port);
 322        }
 323
 324        q->head[priority] = head;
 325}
 326
 327static void __evtchn_fifo_handle_events(unsigned cpu, bool drop)
 328{
 329        struct evtchn_fifo_control_block *control_block;
 330        unsigned long ready;
 331        unsigned q;
 332
 333        control_block = per_cpu(cpu_control_block, cpu);
 334
 335        ready = xchg(&control_block->ready, 0);
 336
 337        while (ready) {
 338                q = find_first_bit(&ready, EVTCHN_FIFO_MAX_QUEUES);
 339                consume_one_event(cpu, control_block, q, &ready, drop);
 340                ready |= xchg(&control_block->ready, 0);
 341        }
 342}
 343
 344static void evtchn_fifo_handle_events(unsigned cpu)
 345{
 346        __evtchn_fifo_handle_events(cpu, false);
 347}
 348
 349static void evtchn_fifo_resume(void)
 350{
 351        unsigned cpu;
 352
 353        for_each_possible_cpu(cpu) {
 354                void *control_block = per_cpu(cpu_control_block, cpu);
 355                int ret;
 356
 357                if (!control_block)
 358                        continue;
 359
 360                /*
 361                 * If this CPU is offline, take the opportunity to
 362                 * free the control block while it is not being
 363                 * used.
 364                 */
 365                if (!cpu_online(cpu)) {
 366                        free_page((unsigned long)control_block);
 367                        per_cpu(cpu_control_block, cpu) = NULL;
 368                        continue;
 369                }
 370
 371                ret = init_control_block(cpu, control_block);
 372                BUG_ON(ret < 0);
 373        }
 374
 375        /*
 376         * The event array starts out as empty again and is extended
 377         * as normal when events are bound.  The existing pages will
 378         * be reused.
 379         */
 380        event_array_pages = 0;
 381}
 382
 383static const struct evtchn_ops evtchn_ops_fifo = {
 384        .max_channels      = evtchn_fifo_max_channels,
 385        .nr_channels       = evtchn_fifo_nr_channels,
 386        .setup             = evtchn_fifo_setup,
 387        .bind_to_cpu       = evtchn_fifo_bind_to_cpu,
 388        .clear_pending     = evtchn_fifo_clear_pending,
 389        .set_pending       = evtchn_fifo_set_pending,
 390        .is_pending        = evtchn_fifo_is_pending,
 391        .test_and_set_mask = evtchn_fifo_test_and_set_mask,
 392        .mask              = evtchn_fifo_mask,
 393        .unmask            = evtchn_fifo_unmask,
 394        .handle_events     = evtchn_fifo_handle_events,
 395        .resume            = evtchn_fifo_resume,
 396};
 397
 398static int evtchn_fifo_alloc_control_block(unsigned cpu)
 399{
 400        void *control_block = NULL;
 401        int ret = -ENOMEM;
 402
 403        control_block = (void *)__get_free_page(GFP_KERNEL);
 404        if (control_block == NULL)
 405                goto error;
 406
 407        ret = init_control_block(cpu, control_block);
 408        if (ret < 0)
 409                goto error;
 410
 411        per_cpu(cpu_control_block, cpu) = control_block;
 412
 413        return 0;
 414
 415  error:
 416        free_page((unsigned long)control_block);
 417        return ret;
 418}
 419
 420static int xen_evtchn_cpu_prepare(unsigned int cpu)
 421{
 422        if (!per_cpu(cpu_control_block, cpu))
 423                return evtchn_fifo_alloc_control_block(cpu);
 424        return 0;
 425}
 426
 427static int xen_evtchn_cpu_dead(unsigned int cpu)
 428{
 429        __evtchn_fifo_handle_events(cpu, true);
 430        return 0;
 431}
 432
 433int __init xen_evtchn_fifo_init(void)
 434{
 435        int cpu = get_cpu();
 436        int ret;
 437
 438        ret = evtchn_fifo_alloc_control_block(cpu);
 439        if (ret < 0)
 440                goto out;
 441
 442        pr_info("Using FIFO-based ABI\n");
 443
 444        evtchn_ops = &evtchn_ops_fifo;
 445
 446        cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
 447                                  "xen/evtchn:prepare",
 448                                  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
 449out:
 450        put_cpu();
 451        return ret;
 452}
 453