qemu/hw/block/virtio-blk.c
<<
>>
Prefs
   1/*
   2 * Virtio Block Device
   3 *
   4 * Copyright IBM, Corp. 2007
   5 *
   6 * Authors:
   7 *  Anthony Liguori   <aliguori@us.ibm.com>
   8 *
   9 * This work is licensed under the terms of the GNU GPL, version 2.  See
  10 * the COPYING file in the top-level directory.
  11 *
  12 */
  13
  14#include "qemu/osdep.h"
  15#include "qapi/error.h"
  16#include "qemu-common.h"
  17#include "qemu/iov.h"
  18#include "qemu/error-report.h"
  19#include "trace.h"
  20#include "hw/block/block.h"
  21#include "sysemu/block-backend.h"
  22#include "sysemu/blockdev.h"
  23#include "hw/virtio/virtio-blk.h"
  24#include "dataplane/virtio-blk.h"
  25#include "scsi/constants.h"
  26#ifdef __linux__
  27# include <scsi/sg.h>
  28#endif
  29#include "hw/virtio/virtio-bus.h"
  30#include "hw/virtio/virtio-access.h"
  31
  32static void virtio_blk_init_request(VirtIOBlock *s, VirtQueue *vq,
  33                                    VirtIOBlockReq *req)
  34{
  35    req->dev = s;
  36    req->vq = vq;
  37    req->qiov.size = 0;
  38    req->in_len = 0;
  39    req->next = NULL;
  40    req->mr_next = NULL;
  41}
  42
  43static void virtio_blk_free_request(VirtIOBlockReq *req)
  44{
  45    g_free(req);
  46}
  47
  48static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
  49{
  50    VirtIOBlock *s = req->dev;
  51    VirtIODevice *vdev = VIRTIO_DEVICE(s);
  52
  53    trace_virtio_blk_req_complete(vdev, req, status);
  54
  55    stb_p(&req->in->status, status);
  56    virtqueue_push(req->vq, &req->elem, req->in_len);
  57    if (s->dataplane_started && !s->dataplane_disabled) {
  58        virtio_blk_data_plane_notify(s->dataplane, req->vq);
  59    } else {
  60        virtio_notify(vdev, req->vq);
  61    }
  62}
  63
  64static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
  65    bool is_read)
  66{
  67    BlockErrorAction action = blk_get_error_action(req->dev->blk,
  68                                                   is_read, error);
  69    VirtIOBlock *s = req->dev;
  70
  71    if (action == BLOCK_ERROR_ACTION_STOP) {
  72        /* Break the link as the next request is going to be parsed from the
  73         * ring again. Otherwise we may end up doing a double completion! */
  74        req->mr_next = NULL;
  75        req->next = s->rq;
  76        s->rq = req;
  77    } else if (action == BLOCK_ERROR_ACTION_REPORT) {
  78        virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
  79        block_acct_failed(blk_get_stats(s->blk), &req->acct);
  80        virtio_blk_free_request(req);
  81    }
  82
  83    blk_error_action(s->blk, action, is_read, error);
  84    return action != BLOCK_ERROR_ACTION_IGNORE;
  85}
  86
  87static void virtio_blk_rw_complete(void *opaque, int ret)
  88{
  89    VirtIOBlockReq *next = opaque;
  90    VirtIOBlock *s = next->dev;
  91    VirtIODevice *vdev = VIRTIO_DEVICE(s);
  92
  93    aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
  94    while (next) {
  95        VirtIOBlockReq *req = next;
  96        next = req->mr_next;
  97        trace_virtio_blk_rw_complete(vdev, req, ret);
  98
  99        if (req->qiov.nalloc != -1) {
 100            /* If nalloc is != 1 req->qiov is a local copy of the original
 101             * external iovec. It was allocated in submit_merged_requests
 102             * to be able to merge requests. */
 103            qemu_iovec_destroy(&req->qiov);
 104        }
 105
 106        if (ret) {
 107            int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
 108            bool is_read = !(p & VIRTIO_BLK_T_OUT);
 109            /* Note that memory may be dirtied on read failure.  If the
 110             * virtio request is not completed here, as is the case for
 111             * BLOCK_ERROR_ACTION_STOP, the memory may not be copied
 112             * correctly during live migration.  While this is ugly,
 113             * it is acceptable because the device is free to write to
 114             * the memory until the request is completed (which will
 115             * happen on the other side of the migration).
 116             */
 117            if (virtio_blk_handle_rw_error(req, -ret, is_read)) {
 118                continue;
 119            }
 120        }
 121
 122        virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
 123        block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
 124        virtio_blk_free_request(req);
 125    }
 126    aio_context_release(blk_get_aio_context(s->conf.conf.blk));
 127}
 128
 129static void virtio_blk_flush_complete(void *opaque, int ret)
 130{
 131    VirtIOBlockReq *req = opaque;
 132    VirtIOBlock *s = req->dev;
 133
 134    aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
 135    if (ret) {
 136        if (virtio_blk_handle_rw_error(req, -ret, 0)) {
 137            goto out;
 138        }
 139    }
 140
 141    virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
 142    block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
 143    virtio_blk_free_request(req);
 144
 145out:
 146    aio_context_release(blk_get_aio_context(s->conf.conf.blk));
 147}
 148
 149#ifdef __linux__
 150
 151typedef struct {
 152    VirtIOBlockReq *req;
 153    struct sg_io_hdr hdr;
 154} VirtIOBlockIoctlReq;
 155
 156static void virtio_blk_ioctl_complete(void *opaque, int status)
 157{
 158    VirtIOBlockIoctlReq *ioctl_req = opaque;
 159    VirtIOBlockReq *req = ioctl_req->req;
 160    VirtIOBlock *s = req->dev;
 161    VirtIODevice *vdev = VIRTIO_DEVICE(s);
 162    struct virtio_scsi_inhdr *scsi;
 163    struct sg_io_hdr *hdr;
 164
 165    scsi = (void *)req->elem.in_sg[req->elem.in_num - 2].iov_base;
 166
 167    if (status) {
 168        status = VIRTIO_BLK_S_UNSUPP;
 169        virtio_stl_p(vdev, &scsi->errors, 255);
 170        goto out;
 171    }
 172
 173    hdr = &ioctl_req->hdr;
 174    /*
 175     * From SCSI-Generic-HOWTO: "Some lower level drivers (e.g. ide-scsi)
 176     * clear the masked_status field [hence status gets cleared too, see
 177     * block/scsi_ioctl.c] even when a CHECK_CONDITION or COMMAND_TERMINATED
 178     * status has occurred.  However they do set DRIVER_SENSE in driver_status
 179     * field. Also a (sb_len_wr > 0) indicates there is a sense buffer.
 180     */
 181    if (hdr->status == 0 && hdr->sb_len_wr > 0) {
 182        hdr->status = CHECK_CONDITION;
 183    }
 184
 185    virtio_stl_p(vdev, &scsi->errors,
 186                 hdr->status | (hdr->msg_status << 8) |
 187                 (hdr->host_status << 16) | (hdr->driver_status << 24));
 188    virtio_stl_p(vdev, &scsi->residual, hdr->resid);
 189    virtio_stl_p(vdev, &scsi->sense_len, hdr->sb_len_wr);
 190    virtio_stl_p(vdev, &scsi->data_len, hdr->dxfer_len);
 191
 192out:
 193    aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
 194    virtio_blk_req_complete(req, status);
 195    virtio_blk_free_request(req);
 196    aio_context_release(blk_get_aio_context(s->conf.conf.blk));
 197    g_free(ioctl_req);
 198}
 199
 200#endif
 201
 202static VirtIOBlockReq *virtio_blk_get_request(VirtIOBlock *s, VirtQueue *vq)
 203{
 204    VirtIOBlockReq *req = virtqueue_pop(vq, sizeof(VirtIOBlockReq));
 205
 206    if (req) {
 207        virtio_blk_init_request(s, vq, req);
 208    }
 209    return req;
 210}
 211
 212static int virtio_blk_handle_scsi_req(VirtIOBlockReq *req)
 213{
 214    int status = VIRTIO_BLK_S_OK;
 215    struct virtio_scsi_inhdr *scsi = NULL;
 216    VirtIODevice *vdev = VIRTIO_DEVICE(req->dev);
 217    VirtQueueElement *elem = &req->elem;
 218    VirtIOBlock *blk = req->dev;
 219
 220#ifdef __linux__
 221    int i;
 222    VirtIOBlockIoctlReq *ioctl_req;
 223    BlockAIOCB *acb;
 224#endif
 225
 226    /*
 227     * We require at least one output segment each for the virtio_blk_outhdr
 228     * and the SCSI command block.
 229     *
 230     * We also at least require the virtio_blk_inhdr, the virtio_scsi_inhdr
 231     * and the sense buffer pointer in the input segments.
 232     */
 233    if (elem->out_num < 2 || elem->in_num < 3) {
 234        status = VIRTIO_BLK_S_IOERR;
 235        goto fail;
 236    }
 237
 238    /*
 239     * The scsi inhdr is placed in the second-to-last input segment, just
 240     * before the regular inhdr.
 241     */
 242    scsi = (void *)elem->in_sg[elem->in_num - 2].iov_base;
 243
 244    if (!blk->conf.scsi) {
 245        status = VIRTIO_BLK_S_UNSUPP;
 246        goto fail;
 247    }
 248
 249    /*
 250     * No support for bidirection commands yet.
 251     */
 252    if (elem->out_num > 2 && elem->in_num > 3) {
 253        status = VIRTIO_BLK_S_UNSUPP;
 254        goto fail;
 255    }
 256
 257#ifdef __linux__
 258    ioctl_req = g_new0(VirtIOBlockIoctlReq, 1);
 259    ioctl_req->req = req;
 260    ioctl_req->hdr.interface_id = 'S';
 261    ioctl_req->hdr.cmd_len = elem->out_sg[1].iov_len;
 262    ioctl_req->hdr.cmdp = elem->out_sg[1].iov_base;
 263    ioctl_req->hdr.dxfer_len = 0;
 264
 265    if (elem->out_num > 2) {
 266        /*
 267         * If there are more than the minimally required 2 output segments
 268         * there is write payload starting from the third iovec.
 269         */
 270        ioctl_req->hdr.dxfer_direction = SG_DXFER_TO_DEV;
 271        ioctl_req->hdr.iovec_count = elem->out_num - 2;
 272
 273        for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
 274            ioctl_req->hdr.dxfer_len += elem->out_sg[i + 2].iov_len;
 275        }
 276
 277        ioctl_req->hdr.dxferp = elem->out_sg + 2;
 278
 279    } else if (elem->in_num > 3) {
 280        /*
 281         * If we have more than 3 input segments the guest wants to actually
 282         * read data.
 283         */
 284        ioctl_req->hdr.dxfer_direction = SG_DXFER_FROM_DEV;
 285        ioctl_req->hdr.iovec_count = elem->in_num - 3;
 286        for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
 287            ioctl_req->hdr.dxfer_len += elem->in_sg[i].iov_len;
 288        }
 289
 290        ioctl_req->hdr.dxferp = elem->in_sg;
 291    } else {
 292        /*
 293         * Some SCSI commands don't actually transfer any data.
 294         */
 295        ioctl_req->hdr.dxfer_direction = SG_DXFER_NONE;
 296    }
 297
 298    ioctl_req->hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base;
 299    ioctl_req->hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len;
 300
 301    acb = blk_aio_ioctl(blk->blk, SG_IO, &ioctl_req->hdr,
 302                        virtio_blk_ioctl_complete, ioctl_req);
 303    if (!acb) {
 304        g_free(ioctl_req);
 305        status = VIRTIO_BLK_S_UNSUPP;
 306        goto fail;
 307    }
 308    return -EINPROGRESS;
 309#else
 310    abort();
 311#endif
 312
 313fail:
 314    /* Just put anything nonzero so that the ioctl fails in the guest.  */
 315    if (scsi) {
 316        virtio_stl_p(vdev, &scsi->errors, 255);
 317    }
 318    return status;
 319}
 320
 321static void virtio_blk_handle_scsi(VirtIOBlockReq *req)
 322{
 323    int status;
 324
 325    status = virtio_blk_handle_scsi_req(req);
 326    if (status != -EINPROGRESS) {
 327        virtio_blk_req_complete(req, status);
 328        virtio_blk_free_request(req);
 329    }
 330}
 331
 332static inline void submit_requests(BlockBackend *blk, MultiReqBuffer *mrb,
 333                                   int start, int num_reqs, int niov)
 334{
 335    QEMUIOVector *qiov = &mrb->reqs[start]->qiov;
 336    int64_t sector_num = mrb->reqs[start]->sector_num;
 337    bool is_write = mrb->is_write;
 338
 339    if (num_reqs > 1) {
 340        int i;
 341        struct iovec *tmp_iov = qiov->iov;
 342        int tmp_niov = qiov->niov;
 343
 344        /* mrb->reqs[start]->qiov was initialized from external so we can't
 345         * modify it here. We need to initialize it locally and then add the
 346         * external iovecs. */
 347        qemu_iovec_init(qiov, niov);
 348
 349        for (i = 0; i < tmp_niov; i++) {
 350            qemu_iovec_add(qiov, tmp_iov[i].iov_base, tmp_iov[i].iov_len);
 351        }
 352
 353        for (i = start + 1; i < start + num_reqs; i++) {
 354            qemu_iovec_concat(qiov, &mrb->reqs[i]->qiov, 0,
 355                              mrb->reqs[i]->qiov.size);
 356            mrb->reqs[i - 1]->mr_next = mrb->reqs[i];
 357        }
 358
 359        trace_virtio_blk_submit_multireq(VIRTIO_DEVICE(mrb->reqs[start]->dev),
 360                                         mrb, start, num_reqs,
 361                                         sector_num << BDRV_SECTOR_BITS,
 362                                         qiov->size, is_write);
 363        block_acct_merge_done(blk_get_stats(blk),
 364                              is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ,
 365                              num_reqs - 1);
 366    }
 367
 368    if (is_write) {
 369        blk_aio_pwritev(blk, sector_num << BDRV_SECTOR_BITS, qiov, 0,
 370                        virtio_blk_rw_complete, mrb->reqs[start]);
 371    } else {
 372        blk_aio_preadv(blk, sector_num << BDRV_SECTOR_BITS, qiov, 0,
 373                       virtio_blk_rw_complete, mrb->reqs[start]);
 374    }
 375}
 376
 377static int multireq_compare(const void *a, const void *b)
 378{
 379    const VirtIOBlockReq *req1 = *(VirtIOBlockReq **)a,
 380                         *req2 = *(VirtIOBlockReq **)b;
 381
 382    /*
 383     * Note that we can't simply subtract sector_num1 from sector_num2
 384     * here as that could overflow the return value.
 385     */
 386    if (req1->sector_num > req2->sector_num) {
 387        return 1;
 388    } else if (req1->sector_num < req2->sector_num) {
 389        return -1;
 390    } else {
 391        return 0;
 392    }
 393}
 394
 395static void virtio_blk_submit_multireq(BlockBackend *blk, MultiReqBuffer *mrb)
 396{
 397    int i = 0, start = 0, num_reqs = 0, niov = 0, nb_sectors = 0;
 398    uint32_t max_transfer;
 399    int64_t sector_num = 0;
 400
 401    if (mrb->num_reqs == 1) {
 402        submit_requests(blk, mrb, 0, 1, -1);
 403        mrb->num_reqs = 0;
 404        return;
 405    }
 406
 407    max_transfer = blk_get_max_transfer(mrb->reqs[0]->dev->blk);
 408
 409    qsort(mrb->reqs, mrb->num_reqs, sizeof(*mrb->reqs),
 410          &multireq_compare);
 411
 412    for (i = 0; i < mrb->num_reqs; i++) {
 413        VirtIOBlockReq *req = mrb->reqs[i];
 414        if (num_reqs > 0) {
 415            /*
 416             * NOTE: We cannot merge the requests in below situations:
 417             * 1. requests are not sequential
 418             * 2. merge would exceed maximum number of IOVs
 419             * 3. merge would exceed maximum transfer length of backend device
 420             */
 421            if (sector_num + nb_sectors != req->sector_num ||
 422                niov > blk_get_max_iov(blk) - req->qiov.niov ||
 423                req->qiov.size > max_transfer ||
 424                nb_sectors > (max_transfer -
 425                              req->qiov.size) / BDRV_SECTOR_SIZE) {
 426                submit_requests(blk, mrb, start, num_reqs, niov);
 427                num_reqs = 0;
 428            }
 429        }
 430
 431        if (num_reqs == 0) {
 432            sector_num = req->sector_num;
 433            nb_sectors = niov = 0;
 434            start = i;
 435        }
 436
 437        nb_sectors += req->qiov.size / BDRV_SECTOR_SIZE;
 438        niov += req->qiov.niov;
 439        num_reqs++;
 440    }
 441
 442    submit_requests(blk, mrb, start, num_reqs, niov);
 443    mrb->num_reqs = 0;
 444}
 445
 446static void virtio_blk_handle_flush(VirtIOBlockReq *req, MultiReqBuffer *mrb)
 447{
 448    block_acct_start(blk_get_stats(req->dev->blk), &req->acct, 0,
 449                     BLOCK_ACCT_FLUSH);
 450
 451    /*
 452     * Make sure all outstanding writes are posted to the backing device.
 453     */
 454    if (mrb->is_write && mrb->num_reqs > 0) {
 455        virtio_blk_submit_multireq(req->dev->blk, mrb);
 456    }
 457    blk_aio_flush(req->dev->blk, virtio_blk_flush_complete, req);
 458}
 459
 460static bool virtio_blk_sect_range_ok(VirtIOBlock *dev,
 461                                     uint64_t sector, size_t size)
 462{
 463    uint64_t nb_sectors = size >> BDRV_SECTOR_BITS;
 464    uint64_t total_sectors;
 465
 466    if (nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
 467        return false;
 468    }
 469    if (sector & dev->sector_mask) {
 470        return false;
 471    }
 472    if (size % dev->conf.conf.logical_block_size) {
 473        return false;
 474    }
 475    blk_get_geometry(dev->blk, &total_sectors);
 476    if (sector > total_sectors || nb_sectors > total_sectors - sector) {
 477        return false;
 478    }
 479    return true;
 480}
 481
 482static int virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
 483{
 484    uint32_t type;
 485    struct iovec *in_iov = req->elem.in_sg;
 486    struct iovec *iov = req->elem.out_sg;
 487    unsigned in_num = req->elem.in_num;
 488    unsigned out_num = req->elem.out_num;
 489    VirtIOBlock *s = req->dev;
 490    VirtIODevice *vdev = VIRTIO_DEVICE(s);
 491
 492    if (req->elem.out_num < 1 || req->elem.in_num < 1) {
 493        virtio_error(vdev, "virtio-blk missing headers");
 494        return -1;
 495    }
 496
 497    if (unlikely(iov_to_buf(iov, out_num, 0, &req->out,
 498                            sizeof(req->out)) != sizeof(req->out))) {
 499        virtio_error(vdev, "virtio-blk request outhdr too short");
 500        return -1;
 501    }
 502
 503    iov_discard_front(&iov, &out_num, sizeof(req->out));
 504
 505    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
 506        virtio_error(vdev, "virtio-blk request inhdr too short");
 507        return -1;
 508    }
 509
 510    /* We always touch the last byte, so just see how big in_iov is.  */
 511    req->in_len = iov_size(in_iov, in_num);
 512    req->in = (void *)in_iov[in_num - 1].iov_base
 513              + in_iov[in_num - 1].iov_len
 514              - sizeof(struct virtio_blk_inhdr);
 515    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
 516
 517    type = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
 518
 519    /* VIRTIO_BLK_T_OUT defines the command direction. VIRTIO_BLK_T_BARRIER
 520     * is an optional flag. Although a guest should not send this flag if
 521     * not negotiated we ignored it in the past. So keep ignoring it. */
 522    switch (type & ~(VIRTIO_BLK_T_OUT | VIRTIO_BLK_T_BARRIER)) {
 523    case VIRTIO_BLK_T_IN:
 524    {
 525        bool is_write = type & VIRTIO_BLK_T_OUT;
 526        req->sector_num = virtio_ldq_p(VIRTIO_DEVICE(req->dev),
 527                                       &req->out.sector);
 528
 529        if (is_write) {
 530            qemu_iovec_init_external(&req->qiov, iov, out_num);
 531            trace_virtio_blk_handle_write(vdev, req, req->sector_num,
 532                                          req->qiov.size / BDRV_SECTOR_SIZE);
 533        } else {
 534            qemu_iovec_init_external(&req->qiov, in_iov, in_num);
 535            trace_virtio_blk_handle_read(vdev, req, req->sector_num,
 536                                         req->qiov.size / BDRV_SECTOR_SIZE);
 537        }
 538
 539        if (!virtio_blk_sect_range_ok(req->dev, req->sector_num,
 540                                      req->qiov.size)) {
 541            virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
 542            block_acct_invalid(blk_get_stats(req->dev->blk),
 543                               is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
 544            virtio_blk_free_request(req);
 545            return 0;
 546        }
 547
 548        block_acct_start(blk_get_stats(req->dev->blk),
 549                         &req->acct, req->qiov.size,
 550                         is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
 551
 552        /* merge would exceed maximum number of requests or IO direction
 553         * changes */
 554        if (mrb->num_reqs > 0 && (mrb->num_reqs == VIRTIO_BLK_MAX_MERGE_REQS ||
 555                                  is_write != mrb->is_write ||
 556                                  !req->dev->conf.request_merging)) {
 557            virtio_blk_submit_multireq(req->dev->blk, mrb);
 558        }
 559
 560        assert(mrb->num_reqs < VIRTIO_BLK_MAX_MERGE_REQS);
 561        mrb->reqs[mrb->num_reqs++] = req;
 562        mrb->is_write = is_write;
 563        break;
 564    }
 565    case VIRTIO_BLK_T_FLUSH:
 566        virtio_blk_handle_flush(req, mrb);
 567        break;
 568    case VIRTIO_BLK_T_SCSI_CMD:
 569        virtio_blk_handle_scsi(req);
 570        break;
 571    case VIRTIO_BLK_T_GET_ID:
 572    {
 573        VirtIOBlock *s = req->dev;
 574
 575        /*
 576         * NB: per existing s/n string convention the string is
 577         * terminated by '\0' only when shorter than buffer.
 578         */
 579        const char *serial = s->conf.serial ? s->conf.serial : "";
 580        size_t size = MIN(strlen(serial) + 1,
 581                          MIN(iov_size(in_iov, in_num),
 582                              VIRTIO_BLK_ID_BYTES));
 583        iov_from_buf(in_iov, in_num, 0, serial, size);
 584        virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
 585        virtio_blk_free_request(req);
 586        break;
 587    }
 588    default:
 589        virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP);
 590        virtio_blk_free_request(req);
 591    }
 592    return 0;
 593}
 594
 595bool virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
 596{
 597    VirtIOBlockReq *req;
 598    MultiReqBuffer mrb = {};
 599    bool progress = false;
 600
 601    aio_context_acquire(blk_get_aio_context(s->blk));
 602    blk_io_plug(s->blk);
 603
 604    do {
 605        virtio_queue_set_notification(vq, 0);
 606
 607        while ((req = virtio_blk_get_request(s, vq))) {
 608            progress = true;
 609            if (virtio_blk_handle_request(req, &mrb)) {
 610                virtqueue_detach_element(req->vq, &req->elem, 0);
 611                virtio_blk_free_request(req);
 612                break;
 613            }
 614        }
 615
 616        virtio_queue_set_notification(vq, 1);
 617    } while (!virtio_queue_empty(vq));
 618
 619    if (mrb.num_reqs) {
 620        virtio_blk_submit_multireq(s->blk, &mrb);
 621    }
 622
 623    blk_io_unplug(s->blk);
 624    aio_context_release(blk_get_aio_context(s->blk));
 625    return progress;
 626}
 627
 628static void virtio_blk_handle_output_do(VirtIOBlock *s, VirtQueue *vq)
 629{
 630    virtio_blk_handle_vq(s, vq);
 631}
 632
 633static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 634{
 635    VirtIOBlock *s = (VirtIOBlock *)vdev;
 636
 637    if (s->dataplane) {
 638        /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
 639         * dataplane here instead of waiting for .set_status().
 640         */
 641        virtio_device_start_ioeventfd(vdev);
 642        if (!s->dataplane_disabled) {
 643            return;
 644        }
 645    }
 646    virtio_blk_handle_output_do(s, vq);
 647}
 648
 649static void virtio_blk_dma_restart_bh(void *opaque)
 650{
 651    VirtIOBlock *s = opaque;
 652    VirtIOBlockReq *req = s->rq;
 653    MultiReqBuffer mrb = {};
 654
 655    qemu_bh_delete(s->bh);
 656    s->bh = NULL;
 657
 658    s->rq = NULL;
 659
 660    aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
 661    while (req) {
 662        VirtIOBlockReq *next = req->next;
 663        if (virtio_blk_handle_request(req, &mrb)) {
 664            /* Device is now broken and won't do any processing until it gets
 665             * reset. Already queued requests will be lost: let's purge them.
 666             */
 667            while (req) {
 668                next = req->next;
 669                virtqueue_detach_element(req->vq, &req->elem, 0);
 670                virtio_blk_free_request(req);
 671                req = next;
 672            }
 673            break;
 674        }
 675        req = next;
 676    }
 677
 678    if (mrb.num_reqs) {
 679        virtio_blk_submit_multireq(s->blk, &mrb);
 680    }
 681    aio_context_release(blk_get_aio_context(s->conf.conf.blk));
 682}
 683
 684static void virtio_blk_dma_restart_cb(void *opaque, int running,
 685                                      RunState state)
 686{
 687    VirtIOBlock *s = opaque;
 688
 689    if (!running) {
 690        return;
 691    }
 692
 693    if (!s->bh) {
 694        s->bh = aio_bh_new(blk_get_aio_context(s->conf.conf.blk),
 695                           virtio_blk_dma_restart_bh, s);
 696        qemu_bh_schedule(s->bh);
 697    }
 698}
 699
 700static void virtio_blk_reset(VirtIODevice *vdev)
 701{
 702    VirtIOBlock *s = VIRTIO_BLK(vdev);
 703    AioContext *ctx;
 704    VirtIOBlockReq *req;
 705
 706    ctx = blk_get_aio_context(s->blk);
 707    aio_context_acquire(ctx);
 708    blk_drain(s->blk);
 709
 710    /* We drop queued requests after blk_drain() because blk_drain() itself can
 711     * produce them. */
 712    while (s->rq) {
 713        req = s->rq;
 714        s->rq = req->next;
 715        virtqueue_detach_element(req->vq, &req->elem, 0);
 716        virtio_blk_free_request(req);
 717    }
 718
 719    aio_context_release(ctx);
 720
 721    assert(!s->dataplane_started);
 722    blk_set_enable_write_cache(s->blk, s->original_wce);
 723}
 724
 725/* coalesce internal state, copy to pci i/o region 0
 726 */
 727static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config)
 728{
 729    VirtIOBlock *s = VIRTIO_BLK(vdev);
 730    BlockConf *conf = &s->conf.conf;
 731    struct virtio_blk_config blkcfg;
 732    uint64_t capacity;
 733    int64_t length;
 734    int blk_size = conf->logical_block_size;
 735
 736    blk_get_geometry(s->blk, &capacity);
 737    memset(&blkcfg, 0, sizeof(blkcfg));
 738    virtio_stq_p(vdev, &blkcfg.capacity, capacity);
 739    virtio_stl_p(vdev, &blkcfg.seg_max, 128 - 2);
 740    virtio_stw_p(vdev, &blkcfg.geometry.cylinders, conf->cyls);
 741    virtio_stl_p(vdev, &blkcfg.blk_size, blk_size);
 742    virtio_stw_p(vdev, &blkcfg.min_io_size, conf->min_io_size / blk_size);
 743    virtio_stw_p(vdev, &blkcfg.opt_io_size, conf->opt_io_size / blk_size);
 744    blkcfg.geometry.heads = conf->heads;
 745    /*
 746     * We must ensure that the block device capacity is a multiple of
 747     * the logical block size. If that is not the case, let's use
 748     * sector_mask to adopt the geometry to have a correct picture.
 749     * For those devices where the capacity is ok for the given geometry
 750     * we don't touch the sector value of the geometry, since some devices
 751     * (like s390 dasd) need a specific value. Here the capacity is already
 752     * cyls*heads*secs*blk_size and the sector value is not block size
 753     * divided by 512 - instead it is the amount of blk_size blocks
 754     * per track (cylinder).
 755     */
 756    length = blk_getlength(s->blk);
 757    if (length > 0 && length / conf->heads / conf->secs % blk_size) {
 758        blkcfg.geometry.sectors = conf->secs & ~s->sector_mask;
 759    } else {
 760        blkcfg.geometry.sectors = conf->secs;
 761    }
 762    blkcfg.size_max = 0;
 763    blkcfg.physical_block_exp = get_physical_block_exp(conf);
 764    blkcfg.alignment_offset = 0;
 765    blkcfg.wce = blk_enable_write_cache(s->blk);
 766    virtio_stw_p(vdev, &blkcfg.num_queues, s->conf.num_queues);
 767    memcpy(config, &blkcfg, sizeof(struct virtio_blk_config));
 768}
 769
 770static void virtio_blk_set_config(VirtIODevice *vdev, const uint8_t *config)
 771{
 772    VirtIOBlock *s = VIRTIO_BLK(vdev);
 773    struct virtio_blk_config blkcfg;
 774
 775    memcpy(&blkcfg, config, sizeof(blkcfg));
 776
 777    aio_context_acquire(blk_get_aio_context(s->blk));
 778    blk_set_enable_write_cache(s->blk, blkcfg.wce != 0);
 779    aio_context_release(blk_get_aio_context(s->blk));
 780}
 781
 782static uint64_t virtio_blk_get_features(VirtIODevice *vdev, uint64_t features,
 783                                        Error **errp)
 784{
 785    VirtIOBlock *s = VIRTIO_BLK(vdev);
 786
 787    virtio_add_feature(&features, VIRTIO_BLK_F_SEG_MAX);
 788    virtio_add_feature(&features, VIRTIO_BLK_F_GEOMETRY);
 789    virtio_add_feature(&features, VIRTIO_BLK_F_TOPOLOGY);
 790    virtio_add_feature(&features, VIRTIO_BLK_F_BLK_SIZE);
 791    if (virtio_has_feature(features, VIRTIO_F_VERSION_1)) {
 792        if (s->conf.scsi) {
 793            error_setg(errp, "Please set scsi=off for virtio-blk devices in order to use virtio 1.0");
 794            return 0;
 795        }
 796    } else {
 797        virtio_clear_feature(&features, VIRTIO_F_ANY_LAYOUT);
 798        virtio_add_feature(&features, VIRTIO_BLK_F_SCSI);
 799    }
 800
 801    if (s->conf.config_wce) {
 802        virtio_add_feature(&features, VIRTIO_BLK_F_CONFIG_WCE);
 803    }
 804    if (blk_enable_write_cache(s->blk)) {
 805        virtio_add_feature(&features, VIRTIO_BLK_F_WCE);
 806    }
 807    if (blk_is_read_only(s->blk)) {
 808        virtio_add_feature(&features, VIRTIO_BLK_F_RO);
 809    }
 810    if (s->conf.num_queues > 1) {
 811        virtio_add_feature(&features, VIRTIO_BLK_F_MQ);
 812    }
 813
 814    return features;
 815}
 816
 817static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
 818{
 819    VirtIOBlock *s = VIRTIO_BLK(vdev);
 820
 821    if (!(status & (VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK))) {
 822        assert(!s->dataplane_started);
 823    }
 824
 825    if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
 826        return;
 827    }
 828
 829    /* A guest that supports VIRTIO_BLK_F_CONFIG_WCE must be able to send
 830     * cache flushes.  Thus, the "auto writethrough" behavior is never
 831     * necessary for guests that support the VIRTIO_BLK_F_CONFIG_WCE feature.
 832     * Leaving it enabled would break the following sequence:
 833     *
 834     *     Guest started with "-drive cache=writethrough"
 835     *     Guest sets status to 0
 836     *     Guest sets DRIVER bit in status field
 837     *     Guest reads host features (WCE=0, CONFIG_WCE=1)
 838     *     Guest writes guest features (WCE=0, CONFIG_WCE=1)
 839     *     Guest writes 1 to the WCE configuration field (writeback mode)
 840     *     Guest sets DRIVER_OK bit in status field
 841     *
 842     * s->blk would erroneously be placed in writethrough mode.
 843     */
 844    if (!virtio_vdev_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) {
 845        aio_context_acquire(blk_get_aio_context(s->blk));
 846        blk_set_enable_write_cache(s->blk,
 847                                   virtio_vdev_has_feature(vdev,
 848                                                           VIRTIO_BLK_F_WCE));
 849        aio_context_release(blk_get_aio_context(s->blk));
 850    }
 851}
 852
 853static void virtio_blk_save_device(VirtIODevice *vdev, QEMUFile *f)
 854{
 855    VirtIOBlock *s = VIRTIO_BLK(vdev);
 856    VirtIOBlockReq *req = s->rq;
 857
 858    while (req) {
 859        qemu_put_sbyte(f, 1);
 860
 861        if (s->conf.num_queues > 1) {
 862            qemu_put_be32(f, virtio_get_queue_index(req->vq));
 863        }
 864
 865        qemu_put_virtqueue_element(f, &req->elem);
 866        req = req->next;
 867    }
 868    qemu_put_sbyte(f, 0);
 869}
 870
 871static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f,
 872                                  int version_id)
 873{
 874    VirtIOBlock *s = VIRTIO_BLK(vdev);
 875
 876    while (qemu_get_sbyte(f)) {
 877        unsigned nvqs = s->conf.num_queues;
 878        unsigned vq_idx = 0;
 879        VirtIOBlockReq *req;
 880
 881        if (nvqs > 1) {
 882            vq_idx = qemu_get_be32(f);
 883
 884            if (vq_idx >= nvqs) {
 885                error_report("Invalid virtqueue index in request list: %#x",
 886                             vq_idx);
 887                return -EINVAL;
 888            }
 889        }
 890
 891        req = qemu_get_virtqueue_element(vdev, f, sizeof(VirtIOBlockReq));
 892        virtio_blk_init_request(s, virtio_get_queue(vdev, vq_idx), req);
 893        req->next = s->rq;
 894        s->rq = req;
 895    }
 896
 897    return 0;
 898}
 899
 900static void virtio_blk_resize(void *opaque)
 901{
 902    VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
 903
 904    virtio_notify_config(vdev);
 905}
 906
 907static const BlockDevOps virtio_block_ops = {
 908    .resize_cb = virtio_blk_resize,
 909};
 910
 911static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
 912{
 913    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 914    VirtIOBlock *s = VIRTIO_BLK(dev);
 915    VirtIOBlkConf *conf = &s->conf;
 916    Error *err = NULL;
 917    unsigned i;
 918
 919    if (!conf->conf.blk) {
 920        error_setg(errp, "drive property not set");
 921        return;
 922    }
 923    if (!blk_is_inserted(conf->conf.blk)) {
 924        error_setg(errp, "Device needs media, but drive is empty");
 925        return;
 926    }
 927    if (!conf->num_queues) {
 928        error_setg(errp, "num-queues property must be larger than 0");
 929        return;
 930    }
 931
 932    blkconf_serial(&conf->conf, &conf->serial);
 933    blkconf_apply_backend_options(&conf->conf,
 934                                  blk_is_read_only(conf->conf.blk), true,
 935                                  &err);
 936    if (err) {
 937        error_propagate(errp, err);
 938        return;
 939    }
 940    s->original_wce = blk_enable_write_cache(conf->conf.blk);
 941    blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, &err);
 942    if (err) {
 943        error_propagate(errp, err);
 944        return;
 945    }
 946    blkconf_blocksizes(&conf->conf);
 947
 948    virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
 949                sizeof(struct virtio_blk_config));
 950
 951    s->blk = conf->conf.blk;
 952    s->rq = NULL;
 953    s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
 954
 955    for (i = 0; i < conf->num_queues; i++) {
 956        virtio_add_queue(vdev, 128, virtio_blk_handle_output);
 957    }
 958    virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
 959    if (err != NULL) {
 960        error_propagate(errp, err);
 961        virtio_cleanup(vdev);
 962        return;
 963    }
 964
 965    s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
 966    blk_set_dev_ops(s->blk, &virtio_block_ops, s);
 967    blk_set_guest_block_size(s->blk, s->conf.conf.logical_block_size);
 968
 969    blk_iostatus_enable(s->blk);
 970}
 971
 972static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
 973{
 974    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 975    VirtIOBlock *s = VIRTIO_BLK(dev);
 976
 977    virtio_blk_data_plane_destroy(s->dataplane);
 978    s->dataplane = NULL;
 979    qemu_del_vm_change_state_handler(s->change);
 980    blockdev_mark_auto_del(s->blk);
 981    virtio_cleanup(vdev);
 982}
 983
 984static void virtio_blk_instance_init(Object *obj)
 985{
 986    VirtIOBlock *s = VIRTIO_BLK(obj);
 987
 988    device_add_bootindex_property(obj, &s->conf.conf.bootindex,
 989                                  "bootindex", "/disk@0,0",
 990                                  DEVICE(obj), NULL);
 991}
 992
 993static const VMStateDescription vmstate_virtio_blk = {
 994    .name = "virtio-blk",
 995    .minimum_version_id = 2,
 996    .version_id = 2,
 997    .fields = (VMStateField[]) {
 998        VMSTATE_VIRTIO_DEVICE,
 999        VMSTATE_END_OF_LIST()
1000    },
1001};
1002
1003static Property virtio_blk_properties[] = {
1004    DEFINE_BLOCK_PROPERTIES(VirtIOBlock, conf.conf),
1005    DEFINE_BLOCK_ERROR_PROPERTIES(VirtIOBlock, conf.conf),
1006    DEFINE_BLOCK_CHS_PROPERTIES(VirtIOBlock, conf.conf),
1007    DEFINE_PROP_STRING("serial", VirtIOBlock, conf.serial),
1008    DEFINE_PROP_BIT("config-wce", VirtIOBlock, conf.config_wce, 0, true),
1009#ifdef __linux__
1010    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, false),
1011#endif
1012    DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
1013                    true),
1014    DEFINE_PROP_UINT16("num-queues", VirtIOBlock, conf.num_queues, 1),
1015    DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD,
1016                     IOThread *),
1017    DEFINE_PROP_END_OF_LIST(),
1018};
1019
1020static void virtio_blk_class_init(ObjectClass *klass, void *data)
1021{
1022    DeviceClass *dc = DEVICE_CLASS(klass);
1023    VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
1024
1025    dc->props = virtio_blk_properties;
1026    dc->vmsd = &vmstate_virtio_blk;
1027    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
1028    vdc->realize = virtio_blk_device_realize;
1029    vdc->unrealize = virtio_blk_device_unrealize;
1030    vdc->get_config = virtio_blk_update_config;
1031    vdc->set_config = virtio_blk_set_config;
1032    vdc->get_features = virtio_blk_get_features;
1033    vdc->set_status = virtio_blk_set_status;
1034    vdc->reset = virtio_blk_reset;
1035    vdc->save = virtio_blk_save_device;
1036    vdc->load = virtio_blk_load_device;
1037    vdc->start_ioeventfd = virtio_blk_data_plane_start;
1038    vdc->stop_ioeventfd = virtio_blk_data_plane_stop;
1039}
1040
1041static const TypeInfo virtio_blk_info = {
1042    .name = TYPE_VIRTIO_BLK,
1043    .parent = TYPE_VIRTIO_DEVICE,
1044    .instance_size = sizeof(VirtIOBlock),
1045    .instance_init = virtio_blk_instance_init,
1046    .class_init = virtio_blk_class_init,
1047};
1048
1049static void virtio_register_types(void)
1050{
1051    type_register_static(&virtio_blk_info);
1052}
1053
1054type_init(virtio_register_types)
1055