LXR linux/drivers/virtio/virtio_pci

   1/*
   2 * Virtio PCI driver - common functionality for all device versions
   3 *
   4 * This module allows virtio devices to be used over a virtual PCI device.
   5 * This can be used with QEMU based VMMs like KVM or Xen.
   6 *
   7 * Copyright IBM Corp. 2007
   8 * Copyright Red Hat, Inc. 2014
   9 *
  10 * Authors:
  11 *  Anthony Liguori  <aliguori@us.ibm.com>
  12 *  Rusty Russell <rusty@rustcorp.com.au>
  13 *  Michael S. Tsirkin <mst@redhat.com>
  14 *
  15 * This work is licensed under the terms of the GNU GPL, version 2 or later.
  16 * See the COPYING file in the top-level directory.
  17 *
  18 */
  19
  20#include "virtio_pci_common.h"
  21
  22static bool force_legacy = false;
  23
  24#if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
  25module_param(force_legacy, bool, 0444);
  26MODULE_PARM_DESC(force_legacy,
  27                 "Force legacy mode for transitional virtio 1 devices");
  28#endif
  29
  30/* wait for pending irq handlers */
  31void vp_synchronize_vectors(struct virtio_device *vdev)
  32{
  33        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  34        int i;
  35
  36        if (vp_dev->intx_enabled)
  37                synchronize_irq(vp_dev->pci_dev->irq);
  38
  39        for (i = 0; i < vp_dev->msix_vectors; ++i)
  40                synchronize_irq(vp_dev->msix_entries[i].vector);
  41}
  42
  43/* the notify function used when creating a virt queue */
  44bool vp_notify(struct virtqueue *vq)
  45{
  46        /* we write the queue's selector into the notification register to
  47         * signal the other end */
  48        iowrite16(vq->index, (void __iomem *)vq->priv);
  49        return true;
  50}
  51
  52/* Handle a configuration change: Tell driver if it wants to know. */
  53static irqreturn_t vp_config_changed(int irq, void *opaque)
  54{
  55        struct virtio_pci_device *vp_dev = opaque;
  56
  57        virtio_config_changed(&vp_dev->vdev);
  58        return IRQ_HANDLED;
  59}
  60
  61/* Notify all virtqueues on an interrupt. */
  62static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
  63{
  64        struct virtio_pci_device *vp_dev = opaque;
  65        struct virtio_pci_vq_info *info;
  66        irqreturn_t ret = IRQ_NONE;
  67        unsigned long flags;
  68
  69        spin_lock_irqsave(&vp_dev->lock, flags);
  70        list_for_each_entry(info, &vp_dev->virtqueues, node) {
  71                if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
  72                        ret = IRQ_HANDLED;
  73        }
  74        spin_unlock_irqrestore(&vp_dev->lock, flags);
  75
  76        return ret;
  77}
  78
  79/* A small wrapper to also acknowledge the interrupt when it's handled.
  80 * I really need an EIO hook for the vring so I can ack the interrupt once we
  81 * know that we'll be handling the IRQ but before we invoke the callback since
  82 * the callback may notify the host which results in the host attempting to
  83 * raise an interrupt that we would then mask once we acknowledged the
  84 * interrupt. */
  85static irqreturn_t vp_interrupt(int irq, void *opaque)
  86{
  87        struct virtio_pci_device *vp_dev = opaque;
  88        u8 isr;
  89
  90        /* reading the ISR has the effect of also clearing it so it's very
  91         * important to save off the value. */
  92        isr = ioread8(vp_dev->isr);
  93
  94        /* It's definitely not us if the ISR was not high */
  95        if (!isr)
  96                return IRQ_NONE;
  97
  98        /* Configuration change?  Tell driver if it wants to know. */
  99        if (isr & VIRTIO_PCI_ISR_CONFIG)
 100                vp_config_changed(irq, opaque);
 101
 102        return vp_vring_interrupt(irq, opaque);
 103}
 104
 105static void vp_free_vectors(struct virtio_device *vdev)
 106{
 107        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 108        int i;
 109
 110        if (vp_dev->intx_enabled) {
 111                free_irq(vp_dev->pci_dev->irq, vp_dev);
 112                vp_dev->intx_enabled = 0;
 113        }
 114
 115        for (i = 0; i < vp_dev->msix_used_vectors; ++i)
 116                free_irq(vp_dev->msix_entries[i].vector, vp_dev);
 117
 118        for (i = 0; i < vp_dev->msix_vectors; i++)
 119                if (vp_dev->msix_affinity_masks[i])
 120                        free_cpumask_var(vp_dev->msix_affinity_masks[i]);
 121
 122        if (vp_dev->msix_enabled) {
 123                /* Disable the vector used for configuration */
 124                vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
 125
 126                pci_disable_msix(vp_dev->pci_dev);
 127                vp_dev->msix_enabled = 0;
 128        }
 129
 130        vp_dev->msix_vectors = 0;
 131        vp_dev->msix_used_vectors = 0;
 132        kfree(vp_dev->msix_names);
 133        vp_dev->msix_names = NULL;
 134        kfree(vp_dev->msix_entries);
 135        vp_dev->msix_entries = NULL;
 136        kfree(vp_dev->msix_affinity_masks);
 137        vp_dev->msix_affinity_masks = NULL;
 138}
 139
 140static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 141                                   bool per_vq_vectors)
 142{
 143        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 144        const char *name = dev_name(&vp_dev->vdev.dev);
 145        unsigned i, v;
 146        int err = -ENOMEM;
 147
 148        vp_dev->msix_vectors = nvectors;
 149
 150        vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
 151                                       GFP_KERNEL);
 152        if (!vp_dev->msix_entries)
 153                goto error;
 154        vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
 155                                     GFP_KERNEL);
 156        if (!vp_dev->msix_names)
 157                goto error;
 158        vp_dev->msix_affinity_masks
 159                = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
 160                          GFP_KERNEL);
 161        if (!vp_dev->msix_affinity_masks)
 162                goto error;
 163        for (i = 0; i < nvectors; ++i)
 164                if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
 165                                        GFP_KERNEL))
 166                        goto error;
 167
 168        for (i = 0; i < nvectors; ++i)
 169                vp_dev->msix_entries[i].entry = i;
 170
 171        err = pci_enable_msix_exact(vp_dev->pci_dev,
 172                                    vp_dev->msix_entries, nvectors);
 173        if (err)
 174                goto error;
 175        vp_dev->msix_enabled = 1;
 176
 177        /* Set the vector used for configuration */
 178        v = vp_dev->msix_used_vectors;
 179        snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 180                 "%s-config", name);
 181        err = request_irq(vp_dev->msix_entries[v].vector,
 182                          vp_config_changed, 0, vp_dev->msix_names[v],
 183                          vp_dev);
 184        if (err)
 185                goto error;
 186        ++vp_dev->msix_used_vectors;
 187
 188        v = vp_dev->config_vector(vp_dev, v);
 189        /* Verify we had enough resources to assign the vector */
 190        if (v == VIRTIO_MSI_NO_VECTOR) {
 191                err = -EBUSY;
 192                goto error;
 193        }
 194
 195        if (!per_vq_vectors) {
 196                /* Shared vector for all VQs */
 197                v = vp_dev->msix_used_vectors;
 198                snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 199                         "%s-virtqueues", name);
 200                err = request_irq(vp_dev->msix_entries[v].vector,
 201                                  vp_vring_interrupt, 0, vp_dev->msix_names[v],
 202                                  vp_dev);
 203                if (err)
 204                        goto error;
 205                ++vp_dev->msix_used_vectors;
 206        }
 207        return 0;
 208error:
 209        vp_free_vectors(vdev);
 210        return err;
 211}
 212
 213static int vp_request_intx(struct virtio_device *vdev)
 214{
 215        int err;
 216        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 217
 218        err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
 219                          IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
 220        if (!err)
 221                vp_dev->intx_enabled = 1;
 222        return err;
 223}
 224
 225static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
 226                                     void (*callback)(struct virtqueue *vq),
 227                                     const char *name,
 228                                     u16 msix_vec)
 229{
 230        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 231        struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
 232        struct virtqueue *vq;
 233        unsigned long flags;
 234
 235        /* fill out our structure that represents an active queue */
 236        if (!info)
 237                return ERR_PTR(-ENOMEM);
 238
 239        vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
 240        if (IS_ERR(vq))
 241                goto out_info;
 242
 243        info->vq = vq;
 244        if (callback) {
 245                spin_lock_irqsave(&vp_dev->lock, flags);
 246                list_add(&info->node, &vp_dev->virtqueues);
 247                spin_unlock_irqrestore(&vp_dev->lock, flags);
 248        } else {
 249                INIT_LIST_HEAD(&info->node);
 250        }
 251
 252        vp_dev->vqs[index] = info;
 253        return vq;
 254
 255out_info:
 256        kfree(info);
 257        return vq;
 258}
 259
 260static void vp_del_vq(struct virtqueue *vq)
 261{
 262        struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
 263        struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 264        unsigned long flags;
 265
 266        spin_lock_irqsave(&vp_dev->lock, flags);
 267        list_del(&info->node);
 268        spin_unlock_irqrestore(&vp_dev->lock, flags);
 269
 270        vp_dev->del_vq(info);
 271        kfree(info);
 272}
 273
 274/* the config->del_vqs() implementation */
 275void vp_del_vqs(struct virtio_device *vdev)
 276{
 277        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 278        struct virtqueue *vq, *n;
 279        struct virtio_pci_vq_info *info;
 280
 281        list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
 282                info = vp_dev->vqs[vq->index];
 283                if (vp_dev->per_vq_vectors &&
 284                        info->msix_vector != VIRTIO_MSI_NO_VECTOR)
 285                        free_irq(vp_dev->msix_entries[info->msix_vector].vector,
 286                                 vq);
 287                vp_del_vq(vq);
 288        }
 289        vp_dev->per_vq_vectors = false;
 290
 291        vp_free_vectors(vdev);
 292        kfree(vp_dev->vqs);
 293        vp_dev->vqs = NULL;
 294}
 295
 296static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 297                              struct virtqueue *vqs[],
 298                              vq_callback_t *callbacks[],
 299                              const char * const names[],
 300                              bool use_msix,
 301                              bool per_vq_vectors)
 302{
 303        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 304        u16 msix_vec;
 305        int i, err, nvectors, allocated_vectors;
 306
 307        vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
 308        if (!vp_dev->vqs)
 309                return -ENOMEM;
 310
 311        if (!use_msix) {
 312                /* Old style: one normal interrupt for change and all vqs. */
 313                err = vp_request_intx(vdev);
 314                if (err)
 315                        goto error_find;
 316        } else {
 317                if (per_vq_vectors) {
 318                        /* Best option: one for change interrupt, one per vq. */
 319                        nvectors = 1;
 320                        for (i = 0; i < nvqs; ++i)
 321                                if (callbacks[i])
 322                                        ++nvectors;
 323                } else {
 324                        /* Second best: one for change, shared for all vqs. */
 325                        nvectors = 2;
 326                }
 327
 328                err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
 329                if (err)
 330                        goto error_find;
 331        }
 332
 333        vp_dev->per_vq_vectors = per_vq_vectors;
 334        allocated_vectors = vp_dev->msix_used_vectors;
 335        for (i = 0; i < nvqs; ++i) {
 336                if (!names[i]) {
 337                        vqs[i] = NULL;
 338                        continue;
 339                } else if (!callbacks[i] || !vp_dev->msix_enabled)
 340                        msix_vec = VIRTIO_MSI_NO_VECTOR;
 341                else if (vp_dev->per_vq_vectors)
 342                        msix_vec = allocated_vectors++;
 343                else
 344                        msix_vec = VP_MSIX_VQ_VECTOR;
 345                vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
 346                if (IS_ERR(vqs[i])) {
 347                        err = PTR_ERR(vqs[i]);
 348                        goto error_find;
 349                }
 350
 351                if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
 352                        continue;
 353
 354                /* allocate per-vq irq if available and necessary */
 355                snprintf(vp_dev->msix_names[msix_vec],
 356                         sizeof *vp_dev->msix_names,
 357                         "%s-%s",
 358                         dev_name(&vp_dev->vdev.dev), names[i]);
 359                err = request_irq(vp_dev->msix_entries[msix_vec].vector,
 360                                  vring_interrupt, 0,
 361                                  vp_dev->msix_names[msix_vec],
 362                                  vqs[i]);
 363                if (err) {
 364                        vp_del_vq(vqs[i]);
 365                        goto error_find;
 366                }
 367        }
 368        return 0;
 369
 370error_find:
 371        vp_del_vqs(vdev);
 372        return err;
 373}
 374
 375/* the config->find_vqs() implementation */
 376int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 377                struct virtqueue *vqs[],
 378                vq_callback_t *callbacks[],
 379                const char * const names[])
 380{
 381        int err;
 382
 383        /* Try MSI-X with one vector per queue. */
 384        err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
 385        if (!err)
 386                return 0;
 387        /* Fallback: MSI-X with one vector for config, one shared for queues. */
 388        err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
 389                                 true, false);
 390        if (!err)
 391                return 0;
 392        /* Finally fall back to regular interrupts. */
 393        return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
 394                                  false, false);
 395}
 396
 397const char *vp_bus_name(struct virtio_device *vdev)
 398{
 399        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 400
 401        return pci_name(vp_dev->pci_dev);
 402}
 403
 404/* Setup the affinity for a virtqueue:
 405 * - force the affinity for per vq vector
 406 * - OR over all affinities for shared MSI
 407 * - ignore the affinity request if we're using INTX
 408 */
 409int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
 410{
 411        struct virtio_device *vdev = vq->vdev;
 412        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 413        struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 414        struct cpumask *mask;
 415        unsigned int irq;
 416
 417        if (!vq->callback)
 418                return -EINVAL;
 419
 420        if (vp_dev->msix_enabled) {
 421                mask = vp_dev->msix_affinity_masks[info->msix_vector];
 422                irq = vp_dev->msix_entries[info->msix_vector].vector;
 423                if (cpu == -1)
 424                        irq_set_affinity_hint(irq, NULL);
 425                else {
 426                        cpumask_set_cpu(cpu, mask);
 427                        irq_set_affinity_hint(irq, mask);
 428                }
 429        }
 430        return 0;
 431}
 432
 433#ifdef CONFIG_PM_SLEEP
 434static int virtio_pci_freeze(struct device *dev)
 435{
 436        struct pci_dev *pci_dev = to_pci_dev(dev);
 437        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 438        int ret;
 439
 440        ret = virtio_device_freeze(&vp_dev->vdev);
 441
 442        if (!ret)
 443                pci_disable_device(pci_dev);
 444        return ret;
 445}
 446
 447static int virtio_pci_restore(struct device *dev)
 448{
 449        struct pci_dev *pci_dev = to_pci_dev(dev);
 450        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 451        int ret;
 452
 453        ret = pci_enable_device(pci_dev);
 454        if (ret)
 455                return ret;
 456
 457        pci_set_master(pci_dev);
 458        return virtio_device_restore(&vp_dev->vdev);
 459}
 460
 461static const struct dev_pm_ops virtio_pci_pm_ops = {
 462        SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
 463};
 464#endif
 465
 466
 467/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
 468static const struct pci_device_id virtio_pci_id_table[] = {
 469        { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) },
 470        { 0 }
 471};
 472
 473MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
 474
 475static void virtio_pci_release_dev(struct device *_d)
 476{
 477        struct virtio_device *vdev = dev_to_virtio(_d);
 478        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 479
 480        /* As struct device is a kobject, it's not safe to
 481         * free the memory (including the reference counter itself)
 482         * until it's release callback. */
 483        kfree(vp_dev);
 484}
 485
 486static int virtio_pci_probe(struct pci_dev *pci_dev,
 487                            const struct pci_device_id *id)
 488{
 489        struct virtio_pci_device *vp_dev;
 490        int rc;
 491
 492        /* allocate our structure and fill it out */
 493        vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
 494        if (!vp_dev)
 495                return -ENOMEM;
 496
 497        pci_set_drvdata(pci_dev, vp_dev);
 498        vp_dev->vdev.dev.parent = &pci_dev->dev;
 499        vp_dev->vdev.dev.release = virtio_pci_release_dev;
 500        vp_dev->pci_dev = pci_dev;
 501        INIT_LIST_HEAD(&vp_dev->virtqueues);
 502        spin_lock_init(&vp_dev->lock);
 503
 504        /* enable the device */
 505        rc = pci_enable_device(pci_dev);
 506        if (rc)
 507                goto err_enable_device;
 508
 509        if (force_legacy) {
 510                rc = virtio_pci_legacy_probe(vp_dev);
 511                /* Also try modern mode if we can't map BAR0 (no IO space). */
 512                if (rc == -ENODEV || rc == -ENOMEM)
 513                        rc = virtio_pci_modern_probe(vp_dev);
 514                if (rc)
 515                        goto err_probe;
 516        } else {
 517                rc = virtio_pci_modern_probe(vp_dev);
 518                if (rc == -ENODEV)
 519                        rc = virtio_pci_legacy_probe(vp_dev);
 520                if (rc)
 521                        goto err_probe;
 522        }
 523
 524        pci_set_master(pci_dev);
 525
 526        rc = register_virtio_device(&vp_dev->vdev);
 527        if (rc)
 528                goto err_register;
 529
 530        return 0;
 531
 532err_register:
 533        if (vp_dev->ioaddr)
 534             virtio_pci_legacy_remove(vp_dev);
 535        else
 536             virtio_pci_modern_remove(vp_dev);
 537err_probe:
 538        pci_disable_device(pci_dev);
 539err_enable_device:
 540        kfree(vp_dev);
 541        return rc;
 542}
 543
 544static void virtio_pci_remove(struct pci_dev *pci_dev)
 545{
 546        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 547        struct device *dev = get_device(&vp_dev->vdev.dev);
 548
 549        unregister_virtio_device(&vp_dev->vdev);
 550
 551        if (vp_dev->ioaddr)
 552                virtio_pci_legacy_remove(vp_dev);
 553        else
 554                virtio_pci_modern_remove(vp_dev);
 555
 556        pci_disable_device(pci_dev);
 557        put_device(dev);
 558}
 559
 560static struct pci_driver virtio_pci_driver = {
 561        .name           = "virtio-pci",
 562        .id_table       = virtio_pci_id_table,
 563        .probe          = virtio_pci_probe,
 564        .remove         = virtio_pci_remove,
 565#ifdef CONFIG_PM_SLEEP
 566        .driver.pm      = &virtio_pci_pm_ops,
 567#endif
 568};
 569
 570module_pci_driver(virtio_pci_driver);
 571
 572MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
 573MODULE_DESCRIPTION("virtio-pci");
 574MODULE_LICENSE("GPL");
 575MODULE_VERSION("1");
 576