qemu/hw/virtio/virtio-crypto.c
<<
>>
Prefs
   1/*
   2 * Virtio crypto Support
   3 *
   4 * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
   5 *
   6 * Authors:
   7 *    Gonglei <arei.gonglei@huawei.com>
   8 *
   9 * This work is licensed under the terms of the GNU GPL, version 2 or
  10 * (at your option) any later version.  See the COPYING file in the
  11 * top-level directory.
  12 */
  13
  14#include "qemu/osdep.h"
  15#include "qemu/iov.h"
  16#include "qemu/main-loop.h"
  17#include "qemu/module.h"
  18#include "qapi/error.h"
  19#include "qemu/error-report.h"
  20
  21#include "hw/virtio/virtio.h"
  22#include "hw/virtio/virtio-crypto.h"
  23#include "hw/qdev-properties.h"
  24#include "hw/virtio/virtio-access.h"
  25#include "standard-headers/linux/virtio_ids.h"
  26#include "sysemu/cryptodev-vhost.h"
  27
  28#define VIRTIO_CRYPTO_VM_VERSION 1
  29
  30/*
  31 * Transfer virtqueue index to crypto queue index.
  32 * The control virtqueue is after the data virtqueues
  33 * so the input value doesn't need to be adjusted
  34 */
  35static inline int virtio_crypto_vq2q(int queue_index)
  36{
  37    return queue_index;
  38}
  39
  40static int
  41virtio_crypto_cipher_session_helper(VirtIODevice *vdev,
  42           CryptoDevBackendSymSessionInfo *info,
  43           struct virtio_crypto_cipher_session_para *cipher_para,
  44           struct iovec **iov, unsigned int *out_num)
  45{
  46    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
  47    unsigned int num = *out_num;
  48
  49    info->cipher_alg = ldl_le_p(&cipher_para->algo);
  50    info->key_len = ldl_le_p(&cipher_para->keylen);
  51    info->direction = ldl_le_p(&cipher_para->op);
  52    DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n",
  53             info->cipher_alg, info->direction);
  54
  55    if (info->key_len > vcrypto->conf.max_cipher_key_len) {
  56        error_report("virtio-crypto length of cipher key is too big: %u",
  57                     info->key_len);
  58        return -VIRTIO_CRYPTO_ERR;
  59    }
  60    /* Get cipher key */
  61    if (info->key_len > 0) {
  62        size_t s;
  63        DPRINTF("keylen=%" PRIu32 "\n", info->key_len);
  64
  65        info->cipher_key = g_malloc(info->key_len);
  66        s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len);
  67        if (unlikely(s != info->key_len)) {
  68            virtio_error(vdev, "virtio-crypto cipher key incorrect");
  69            return -EFAULT;
  70        }
  71        iov_discard_front(iov, &num, info->key_len);
  72        *out_num = num;
  73    }
  74
  75    return 0;
  76}
  77
  78static int64_t
  79virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto,
  80               struct virtio_crypto_sym_create_session_req *sess_req,
  81               uint32_t queue_id,
  82               uint32_t opcode,
  83               struct iovec *iov, unsigned int out_num)
  84{
  85    VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
  86    CryptoDevBackendSymSessionInfo info;
  87    int64_t session_id;
  88    int queue_index;
  89    uint32_t op_type;
  90    Error *local_err = NULL;
  91    int ret;
  92
  93    memset(&info, 0, sizeof(info));
  94    op_type = ldl_le_p(&sess_req->op_type);
  95    info.op_type = op_type;
  96    info.op_code = opcode;
  97
  98    if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
  99        ret = virtio_crypto_cipher_session_helper(vdev, &info,
 100                           &sess_req->u.cipher.para,
 101                           &iov, &out_num);
 102        if (ret < 0) {
 103            goto err;
 104        }
 105    } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 106        size_t s;
 107        /* cipher part */
 108        ret = virtio_crypto_cipher_session_helper(vdev, &info,
 109                           &sess_req->u.chain.para.cipher_param,
 110                           &iov, &out_num);
 111        if (ret < 0) {
 112            goto err;
 113        }
 114        /* hash part */
 115        info.alg_chain_order = ldl_le_p(
 116                                     &sess_req->u.chain.para.alg_chain_order);
 117        info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len);
 118        info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode);
 119        if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) {
 120            info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo);
 121            info.auth_key_len = ldl_le_p(
 122                             &sess_req->u.chain.para.u.mac_param.auth_key_len);
 123            info.hash_result_len = ldl_le_p(
 124                           &sess_req->u.chain.para.u.mac_param.hash_result_len);
 125            if (info.auth_key_len > vcrypto->conf.max_auth_key_len) {
 126                error_report("virtio-crypto length of auth key is too big: %u",
 127                             info.auth_key_len);
 128                ret = -VIRTIO_CRYPTO_ERR;
 129                goto err;
 130            }
 131            /* get auth key */
 132            if (info.auth_key_len > 0) {
 133                DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len);
 134                info.auth_key = g_malloc(info.auth_key_len);
 135                s = iov_to_buf(iov, out_num, 0, info.auth_key,
 136                               info.auth_key_len);
 137                if (unlikely(s != info.auth_key_len)) {
 138                    virtio_error(vdev,
 139                          "virtio-crypto authenticated key incorrect");
 140                    ret = -EFAULT;
 141                    goto err;
 142                }
 143                iov_discard_front(&iov, &out_num, info.auth_key_len);
 144            }
 145        } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) {
 146            info.hash_alg = ldl_le_p(
 147                             &sess_req->u.chain.para.u.hash_param.algo);
 148            info.hash_result_len = ldl_le_p(
 149                        &sess_req->u.chain.para.u.hash_param.hash_result_len);
 150        } else {
 151            /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
 152            error_report("unsupported hash mode");
 153            ret = -VIRTIO_CRYPTO_NOTSUPP;
 154            goto err;
 155        }
 156    } else {
 157        /* VIRTIO_CRYPTO_SYM_OP_NONE */
 158        error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE");
 159        ret = -VIRTIO_CRYPTO_NOTSUPP;
 160        goto err;
 161    }
 162
 163    queue_index = virtio_crypto_vq2q(queue_id);
 164    session_id = cryptodev_backend_sym_create_session(
 165                                     vcrypto->cryptodev,
 166                                     &info, queue_index, &local_err);
 167    if (session_id >= 0) {
 168        DPRINTF("create session_id=%" PRIu64 " successfully\n",
 169                session_id);
 170
 171        ret = session_id;
 172    } else {
 173        if (local_err) {
 174            error_report_err(local_err);
 175        }
 176        ret = -VIRTIO_CRYPTO_ERR;
 177    }
 178
 179err:
 180    g_free(info.cipher_key);
 181    g_free(info.auth_key);
 182    return ret;
 183}
 184
 185static uint8_t
 186virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto,
 187         struct virtio_crypto_destroy_session_req *close_sess_req,
 188         uint32_t queue_id)
 189{
 190    int ret;
 191    uint64_t session_id;
 192    uint32_t status;
 193    Error *local_err = NULL;
 194
 195    session_id = ldq_le_p(&close_sess_req->session_id);
 196    DPRINTF("close session, id=%" PRIu64 "\n", session_id);
 197
 198    ret = cryptodev_backend_sym_close_session(
 199              vcrypto->cryptodev, session_id, queue_id, &local_err);
 200    if (ret == 0) {
 201        status = VIRTIO_CRYPTO_OK;
 202    } else {
 203        if (local_err) {
 204            error_report_err(local_err);
 205        } else {
 206            error_report("destroy session failed");
 207        }
 208        status = VIRTIO_CRYPTO_ERR;
 209    }
 210
 211    return status;
 212}
 213
 214static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 215{
 216    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 217    struct virtio_crypto_op_ctrl_req ctrl;
 218    VirtQueueElement *elem;
 219    struct iovec *in_iov;
 220    struct iovec *out_iov;
 221    unsigned in_num;
 222    unsigned out_num;
 223    uint32_t queue_id;
 224    uint32_t opcode;
 225    struct virtio_crypto_session_input input;
 226    int64_t session_id;
 227    uint8_t status;
 228    size_t s;
 229
 230    for (;;) {
 231        g_autofree struct iovec *out_iov_copy = NULL;
 232
 233        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 234        if (!elem) {
 235            break;
 236        }
 237        if (elem->out_num < 1 || elem->in_num < 1) {
 238            virtio_error(vdev, "virtio-crypto ctrl missing headers");
 239            virtqueue_detach_element(vq, elem, 0);
 240            g_free(elem);
 241            break;
 242        }
 243
 244        out_num = elem->out_num;
 245        out_iov_copy = g_memdup(elem->out_sg, sizeof(out_iov[0]) * out_num);
 246        out_iov = out_iov_copy;
 247
 248        in_num = elem->in_num;
 249        in_iov = elem->in_sg;
 250
 251        if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl))
 252                    != sizeof(ctrl))) {
 253            virtio_error(vdev, "virtio-crypto request ctrl_hdr too short");
 254            virtqueue_detach_element(vq, elem, 0);
 255            g_free(elem);
 256            break;
 257        }
 258        iov_discard_front(&out_iov, &out_num, sizeof(ctrl));
 259
 260        opcode = ldl_le_p(&ctrl.header.opcode);
 261        queue_id = ldl_le_p(&ctrl.header.queue_id);
 262
 263        switch (opcode) {
 264        case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION:
 265            memset(&input, 0, sizeof(input));
 266            session_id = virtio_crypto_create_sym_session(vcrypto,
 267                             &ctrl.u.sym_create_session,
 268                             queue_id, opcode,
 269                             out_iov, out_num);
 270            /* Serious errors, need to reset virtio crypto device */
 271            if (session_id == -EFAULT) {
 272                virtqueue_detach_element(vq, elem, 0);
 273                break;
 274            } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) {
 275                stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 276            } else if (session_id == -VIRTIO_CRYPTO_ERR) {
 277                stl_le_p(&input.status, VIRTIO_CRYPTO_ERR);
 278            } else {
 279                /* Set the session id */
 280                stq_le_p(&input.session_id, session_id);
 281                stl_le_p(&input.status, VIRTIO_CRYPTO_OK);
 282            }
 283
 284            s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 285            if (unlikely(s != sizeof(input))) {
 286                virtio_error(vdev, "virtio-crypto input incorrect");
 287                virtqueue_detach_element(vq, elem, 0);
 288                break;
 289            }
 290            virtqueue_push(vq, elem, sizeof(input));
 291            virtio_notify(vdev, vq);
 292            break;
 293        case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION:
 294        case VIRTIO_CRYPTO_HASH_DESTROY_SESSION:
 295        case VIRTIO_CRYPTO_MAC_DESTROY_SESSION:
 296        case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION:
 297            status = virtio_crypto_handle_close_session(vcrypto,
 298                   &ctrl.u.destroy_session, queue_id);
 299            /* The status only occupy one byte, we can directly use it */
 300            s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status));
 301            if (unlikely(s != sizeof(status))) {
 302                virtio_error(vdev, "virtio-crypto status incorrect");
 303                virtqueue_detach_element(vq, elem, 0);
 304                break;
 305            }
 306            virtqueue_push(vq, elem, sizeof(status));
 307            virtio_notify(vdev, vq);
 308            break;
 309        case VIRTIO_CRYPTO_HASH_CREATE_SESSION:
 310        case VIRTIO_CRYPTO_MAC_CREATE_SESSION:
 311        case VIRTIO_CRYPTO_AEAD_CREATE_SESSION:
 312        default:
 313            error_report("virtio-crypto unsupported ctrl opcode: %d", opcode);
 314            memset(&input, 0, sizeof(input));
 315            stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 316            s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 317            if (unlikely(s != sizeof(input))) {
 318                virtio_error(vdev, "virtio-crypto input incorrect");
 319                virtqueue_detach_element(vq, elem, 0);
 320                break;
 321            }
 322            virtqueue_push(vq, elem, sizeof(input));
 323            virtio_notify(vdev, vq);
 324
 325            break;
 326        } /* end switch case */
 327
 328        g_free(elem);
 329    } /* end for loop */
 330}
 331
 332static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq,
 333                                VirtIOCryptoReq *req)
 334{
 335    req->vcrypto = vcrypto;
 336    req->vq = vq;
 337    req->in = NULL;
 338    req->in_iov = NULL;
 339    req->in_num = 0;
 340    req->in_len = 0;
 341    req->flags = CRYPTODEV_BACKEND_ALG__MAX;
 342    req->u.sym_op_info = NULL;
 343}
 344
 345static void virtio_crypto_free_request(VirtIOCryptoReq *req)
 346{
 347    if (req) {
 348        if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 349            size_t max_len;
 350            CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info;
 351
 352            max_len = op_info->iv_len +
 353                      op_info->aad_len +
 354                      op_info->src_len +
 355                      op_info->dst_len +
 356                      op_info->digest_result_len;
 357
 358            /* Zeroize and free request data structure */
 359            memset(op_info, 0, sizeof(*op_info) + max_len);
 360            g_free(op_info);
 361        }
 362        g_free(req);
 363    }
 364}
 365
 366static void
 367virtio_crypto_sym_input_data_helper(VirtIODevice *vdev,
 368                VirtIOCryptoReq *req,
 369                uint32_t status,
 370                CryptoDevBackendSymOpInfo *sym_op_info)
 371{
 372    size_t s, len;
 373
 374    if (status != VIRTIO_CRYPTO_OK) {
 375        return;
 376    }
 377
 378    len = sym_op_info->src_len;
 379    /* Save the cipher result */
 380    s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len);
 381    if (s != len) {
 382        virtio_error(vdev, "virtio-crypto dest data incorrect");
 383        return;
 384    }
 385
 386    iov_discard_front(&req->in_iov, &req->in_num, len);
 387
 388    if (sym_op_info->op_type ==
 389                      VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 390        /* Save the digest result */
 391        s = iov_from_buf(req->in_iov, req->in_num, 0,
 392                         sym_op_info->digest_result,
 393                         sym_op_info->digest_result_len);
 394        if (s != sym_op_info->digest_result_len) {
 395            virtio_error(vdev, "virtio-crypto digest result incorrect");
 396        }
 397    }
 398}
 399
 400static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status)
 401{
 402    VirtIOCrypto *vcrypto = req->vcrypto;
 403    VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 404
 405    if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 406        virtio_crypto_sym_input_data_helper(vdev, req, status,
 407                                            req->u.sym_op_info);
 408    }
 409    stb_p(&req->in->status, status);
 410    virtqueue_push(req->vq, &req->elem, req->in_len);
 411    virtio_notify(vdev, req->vq);
 412}
 413
 414static VirtIOCryptoReq *
 415virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq)
 416{
 417    VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq));
 418
 419    if (req) {
 420        virtio_crypto_init_request(s, vq, req);
 421    }
 422    return req;
 423}
 424
 425static CryptoDevBackendSymOpInfo *
 426virtio_crypto_sym_op_helper(VirtIODevice *vdev,
 427           struct virtio_crypto_cipher_para *cipher_para,
 428           struct virtio_crypto_alg_chain_data_para *alg_chain_para,
 429           struct iovec *iov, unsigned int out_num)
 430{
 431    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 432    CryptoDevBackendSymOpInfo *op_info;
 433    uint32_t src_len = 0, dst_len = 0;
 434    uint32_t iv_len = 0;
 435    uint32_t aad_len = 0, hash_result_len = 0;
 436    uint32_t hash_start_src_offset = 0, len_to_hash = 0;
 437    uint32_t cipher_start_src_offset = 0, len_to_cipher = 0;
 438
 439    uint64_t max_len, curr_size = 0;
 440    size_t s;
 441
 442    /* Plain cipher */
 443    if (cipher_para) {
 444        iv_len = ldl_le_p(&cipher_para->iv_len);
 445        src_len = ldl_le_p(&cipher_para->src_data_len);
 446        dst_len = ldl_le_p(&cipher_para->dst_data_len);
 447    } else if (alg_chain_para) { /* Algorithm chain */
 448        iv_len = ldl_le_p(&alg_chain_para->iv_len);
 449        src_len = ldl_le_p(&alg_chain_para->src_data_len);
 450        dst_len = ldl_le_p(&alg_chain_para->dst_data_len);
 451
 452        aad_len = ldl_le_p(&alg_chain_para->aad_len);
 453        hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len);
 454        hash_start_src_offset = ldl_le_p(
 455                         &alg_chain_para->hash_start_src_offset);
 456        cipher_start_src_offset = ldl_le_p(
 457                         &alg_chain_para->cipher_start_src_offset);
 458        len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher);
 459        len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash);
 460    } else {
 461        return NULL;
 462    }
 463
 464    max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len;
 465    if (unlikely(max_len > vcrypto->conf.max_size)) {
 466        virtio_error(vdev, "virtio-crypto too big length");
 467        return NULL;
 468    }
 469
 470    op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len);
 471    op_info->iv_len = iv_len;
 472    op_info->src_len = src_len;
 473    op_info->dst_len = dst_len;
 474    op_info->aad_len = aad_len;
 475    op_info->digest_result_len = hash_result_len;
 476    op_info->hash_start_src_offset = hash_start_src_offset;
 477    op_info->len_to_hash = len_to_hash;
 478    op_info->cipher_start_src_offset = cipher_start_src_offset;
 479    op_info->len_to_cipher = len_to_cipher;
 480    /* Handle the initilization vector */
 481    if (op_info->iv_len > 0) {
 482        DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len);
 483        op_info->iv = op_info->data + curr_size;
 484
 485        s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len);
 486        if (unlikely(s != op_info->iv_len)) {
 487            virtio_error(vdev, "virtio-crypto iv incorrect");
 488            goto err;
 489        }
 490        iov_discard_front(&iov, &out_num, op_info->iv_len);
 491        curr_size += op_info->iv_len;
 492    }
 493
 494    /* Handle additional authentication data if exists */
 495    if (op_info->aad_len > 0) {
 496        DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len);
 497        op_info->aad_data = op_info->data + curr_size;
 498
 499        s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len);
 500        if (unlikely(s != op_info->aad_len)) {
 501            virtio_error(vdev, "virtio-crypto additional auth data incorrect");
 502            goto err;
 503        }
 504        iov_discard_front(&iov, &out_num, op_info->aad_len);
 505
 506        curr_size += op_info->aad_len;
 507    }
 508
 509    /* Handle the source data */
 510    if (op_info->src_len > 0) {
 511        DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len);
 512        op_info->src = op_info->data + curr_size;
 513
 514        s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len);
 515        if (unlikely(s != op_info->src_len)) {
 516            virtio_error(vdev, "virtio-crypto source data incorrect");
 517            goto err;
 518        }
 519        iov_discard_front(&iov, &out_num, op_info->src_len);
 520
 521        curr_size += op_info->src_len;
 522    }
 523
 524    /* Handle the destination data */
 525    op_info->dst = op_info->data + curr_size;
 526    curr_size += op_info->dst_len;
 527
 528    DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len);
 529
 530    /* Handle the hash digest result */
 531    if (hash_result_len > 0) {
 532        DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len);
 533        op_info->digest_result = op_info->data + curr_size;
 534    }
 535
 536    return op_info;
 537
 538err:
 539    g_free(op_info);
 540    return NULL;
 541}
 542
 543static int
 544virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto,
 545               struct virtio_crypto_sym_data_req *req,
 546               CryptoDevBackendSymOpInfo **sym_op_info,
 547               struct iovec *iov, unsigned int out_num)
 548{
 549    VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 550    uint32_t op_type;
 551    CryptoDevBackendSymOpInfo *op_info;
 552
 553    op_type = ldl_le_p(&req->op_type);
 554
 555    if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
 556        op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para,
 557                                              NULL, iov, out_num);
 558        if (!op_info) {
 559            return -EFAULT;
 560        }
 561        op_info->op_type = op_type;
 562    } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 563        op_info = virtio_crypto_sym_op_helper(vdev, NULL,
 564                                              &req->u.chain.para,
 565                                              iov, out_num);
 566        if (!op_info) {
 567            return -EFAULT;
 568        }
 569        op_info->op_type = op_type;
 570    } else {
 571        /* VIRTIO_CRYPTO_SYM_OP_NONE */
 572        error_report("virtio-crypto unsupported cipher type");
 573        return -VIRTIO_CRYPTO_NOTSUPP;
 574    }
 575
 576    *sym_op_info = op_info;
 577
 578    return 0;
 579}
 580
 581static int
 582virtio_crypto_handle_request(VirtIOCryptoReq *request)
 583{
 584    VirtIOCrypto *vcrypto = request->vcrypto;
 585    VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 586    VirtQueueElement *elem = &request->elem;
 587    int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq));
 588    struct virtio_crypto_op_data_req req;
 589    int ret;
 590    g_autofree struct iovec *in_iov_copy = NULL;
 591    g_autofree struct iovec *out_iov_copy = NULL;
 592    struct iovec *in_iov;
 593    struct iovec *out_iov;
 594    unsigned in_num;
 595    unsigned out_num;
 596    uint32_t opcode;
 597    uint8_t status = VIRTIO_CRYPTO_ERR;
 598    uint64_t session_id;
 599    CryptoDevBackendSymOpInfo *sym_op_info = NULL;
 600    Error *local_err = NULL;
 601
 602    if (elem->out_num < 1 || elem->in_num < 1) {
 603        virtio_error(vdev, "virtio-crypto dataq missing headers");
 604        return -1;
 605    }
 606
 607    out_num = elem->out_num;
 608    out_iov_copy = g_memdup(elem->out_sg, sizeof(out_iov[0]) * out_num);
 609    out_iov = out_iov_copy;
 610
 611    in_num = elem->in_num;
 612    in_iov_copy = g_memdup(elem->in_sg, sizeof(in_iov[0]) * in_num);
 613    in_iov = in_iov_copy;
 614
 615    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req))
 616                != sizeof(req))) {
 617        virtio_error(vdev, "virtio-crypto request outhdr too short");
 618        return -1;
 619    }
 620    iov_discard_front(&out_iov, &out_num, sizeof(req));
 621
 622    if (in_iov[in_num - 1].iov_len <
 623            sizeof(struct virtio_crypto_inhdr)) {
 624        virtio_error(vdev, "virtio-crypto request inhdr too short");
 625        return -1;
 626    }
 627    /* We always touch the last byte, so just see how big in_iov is. */
 628    request->in_len = iov_size(in_iov, in_num);
 629    request->in = (void *)in_iov[in_num - 1].iov_base
 630              + in_iov[in_num - 1].iov_len
 631              - sizeof(struct virtio_crypto_inhdr);
 632    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr));
 633
 634    /*
 635     * The length of operation result, including dest_data
 636     * and digest_result if exists.
 637     */
 638    request->in_num = in_num;
 639    request->in_iov = in_iov;
 640
 641    opcode = ldl_le_p(&req.header.opcode);
 642    session_id = ldq_le_p(&req.header.session_id);
 643
 644    switch (opcode) {
 645    case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
 646    case VIRTIO_CRYPTO_CIPHER_DECRYPT:
 647        ret = virtio_crypto_handle_sym_req(vcrypto,
 648                         &req.u.sym_req,
 649                         &sym_op_info,
 650                         out_iov, out_num);
 651        /* Serious errors, need to reset virtio crypto device */
 652        if (ret == -EFAULT) {
 653            return -1;
 654        } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) {
 655            virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 656            virtio_crypto_free_request(request);
 657        } else {
 658            sym_op_info->session_id = session_id;
 659
 660            /* Set request's parameter */
 661            request->flags = CRYPTODEV_BACKEND_ALG_SYM;
 662            request->u.sym_op_info = sym_op_info;
 663            ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev,
 664                                    request, queue_index, &local_err);
 665            if (ret < 0) {
 666                status = -ret;
 667                if (local_err) {
 668                    error_report_err(local_err);
 669                }
 670            } else { /* ret == VIRTIO_CRYPTO_OK */
 671                status = ret;
 672            }
 673            virtio_crypto_req_complete(request, status);
 674            virtio_crypto_free_request(request);
 675        }
 676        break;
 677    case VIRTIO_CRYPTO_HASH:
 678    case VIRTIO_CRYPTO_MAC:
 679    case VIRTIO_CRYPTO_AEAD_ENCRYPT:
 680    case VIRTIO_CRYPTO_AEAD_DECRYPT:
 681    default:
 682        error_report("virtio-crypto unsupported dataq opcode: %u",
 683                     opcode);
 684        virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 685        virtio_crypto_free_request(request);
 686    }
 687
 688    return 0;
 689}
 690
 691static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq)
 692{
 693    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 694    VirtIOCryptoReq *req;
 695
 696    while ((req = virtio_crypto_get_request(vcrypto, vq))) {
 697        if (virtio_crypto_handle_request(req) < 0) {
 698            virtqueue_detach_element(req->vq, &req->elem, 0);
 699            virtio_crypto_free_request(req);
 700            break;
 701        }
 702    }
 703}
 704
 705static void virtio_crypto_dataq_bh(void *opaque)
 706{
 707    VirtIOCryptoQueue *q = opaque;
 708    VirtIOCrypto *vcrypto = q->vcrypto;
 709    VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 710
 711    /* This happens when device was stopped but BH wasn't. */
 712    if (!vdev->vm_running) {
 713        return;
 714    }
 715
 716    /* Just in case the driver is not ready on more */
 717    if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
 718        return;
 719    }
 720
 721    for (;;) {
 722        virtio_crypto_handle_dataq(vdev, q->dataq);
 723        virtio_queue_set_notification(q->dataq, 1);
 724
 725        /* Are we done or did the guest add more buffers? */
 726        if (virtio_queue_empty(q->dataq)) {
 727            break;
 728        }
 729
 730        virtio_queue_set_notification(q->dataq, 0);
 731    }
 732}
 733
 734static void
 735virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq)
 736{
 737    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 738    VirtIOCryptoQueue *q =
 739         &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))];
 740
 741    /* This happens when device was stopped but VCPU wasn't. */
 742    if (!vdev->vm_running) {
 743        return;
 744    }
 745    virtio_queue_set_notification(vq, 0);
 746    qemu_bh_schedule(q->dataq_bh);
 747}
 748
 749static uint64_t virtio_crypto_get_features(VirtIODevice *vdev,
 750                                           uint64_t features,
 751                                           Error **errp)
 752{
 753    return features;
 754}
 755
 756static void virtio_crypto_reset(VirtIODevice *vdev)
 757{
 758    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 759    /* multiqueue is disabled by default */
 760    vcrypto->curr_queues = 1;
 761    if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 762        vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 763    } else {
 764        vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 765    }
 766}
 767
 768static void virtio_crypto_init_config(VirtIODevice *vdev)
 769{
 770    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 771
 772    vcrypto->conf.crypto_services =
 773                     vcrypto->conf.cryptodev->conf.crypto_services;
 774    vcrypto->conf.cipher_algo_l =
 775                     vcrypto->conf.cryptodev->conf.cipher_algo_l;
 776    vcrypto->conf.cipher_algo_h =
 777                     vcrypto->conf.cryptodev->conf.cipher_algo_h;
 778    vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo;
 779    vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l;
 780    vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h;
 781    vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo;
 782    vcrypto->conf.max_cipher_key_len =
 783                  vcrypto->conf.cryptodev->conf.max_cipher_key_len;
 784    vcrypto->conf.max_auth_key_len =
 785                  vcrypto->conf.cryptodev->conf.max_auth_key_len;
 786    vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size;
 787}
 788
 789static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
 790{
 791    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 792    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 793    int i;
 794
 795    vcrypto->cryptodev = vcrypto->conf.cryptodev;
 796    if (vcrypto->cryptodev == NULL) {
 797        error_setg(errp, "'cryptodev' parameter expects a valid object");
 798        return;
 799    } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) {
 800        error_setg(errp, "can't use already used cryptodev backend: %s",
 801                   object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev)));
 802        return;
 803    }
 804
 805    vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1);
 806    if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) {
 807        error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
 808                   "must be a positive integer less than %d.",
 809                   vcrypto->max_queues, VIRTIO_QUEUE_MAX);
 810        return;
 811    }
 812
 813    virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size);
 814    vcrypto->curr_queues = 1;
 815    vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues);
 816    for (i = 0; i < vcrypto->max_queues; i++) {
 817        vcrypto->vqs[i].dataq =
 818                 virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh);
 819        vcrypto->vqs[i].dataq_bh =
 820                 qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]);
 821        vcrypto->vqs[i].vcrypto = vcrypto;
 822    }
 823
 824    vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl);
 825    if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 826        vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 827    } else {
 828        vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 829    }
 830
 831    virtio_crypto_init_config(vdev);
 832    cryptodev_backend_set_used(vcrypto->cryptodev, true);
 833}
 834
 835static void virtio_crypto_device_unrealize(DeviceState *dev)
 836{
 837    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 838    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 839    VirtIOCryptoQueue *q;
 840    int i, max_queues;
 841
 842    max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1;
 843    for (i = 0; i < max_queues; i++) {
 844        virtio_delete_queue(vcrypto->vqs[i].dataq);
 845        q = &vcrypto->vqs[i];
 846        qemu_bh_delete(q->dataq_bh);
 847    }
 848
 849    g_free(vcrypto->vqs);
 850    virtio_delete_queue(vcrypto->ctrl_vq);
 851
 852    virtio_cleanup(vdev);
 853    cryptodev_backend_set_used(vcrypto->cryptodev, false);
 854}
 855
 856static const VMStateDescription vmstate_virtio_crypto = {
 857    .name = "virtio-crypto",
 858    .unmigratable = 1,
 859    .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION,
 860    .version_id = VIRTIO_CRYPTO_VM_VERSION,
 861    .fields = (VMStateField[]) {
 862        VMSTATE_VIRTIO_DEVICE,
 863        VMSTATE_END_OF_LIST()
 864    },
 865};
 866
 867static Property virtio_crypto_properties[] = {
 868    DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev,
 869                     TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *),
 870    DEFINE_PROP_END_OF_LIST(),
 871};
 872
 873static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config)
 874{
 875    VirtIOCrypto *c = VIRTIO_CRYPTO(vdev);
 876    struct virtio_crypto_config crypto_cfg = {};
 877
 878    /*
 879     * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE,
 880     * so we can use LE accessors directly.
 881     */
 882    stl_le_p(&crypto_cfg.status, c->status);
 883    stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues);
 884    stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services);
 885    stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l);
 886    stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h);
 887    stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo);
 888    stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l);
 889    stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h);
 890    stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo);
 891    stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len);
 892    stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len);
 893    stq_le_p(&crypto_cfg.max_size, c->conf.max_size);
 894
 895    memcpy(config, &crypto_cfg, c->config_size);
 896}
 897
 898static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status)
 899{
 900    VirtIODevice *vdev = VIRTIO_DEVICE(c);
 901    return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 902        (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running;
 903}
 904
 905static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status)
 906{
 907    VirtIODevice *vdev = VIRTIO_DEVICE(c);
 908    int queues = c->multiqueue ? c->max_queues : 1;
 909    CryptoDevBackend *b = c->cryptodev;
 910    CryptoDevBackendClient *cc = b->conf.peers.ccs[0];
 911
 912    if (!cryptodev_get_vhost(cc, b, 0)) {
 913        return;
 914    }
 915
 916    if ((virtio_crypto_started(c, status)) == !!c->vhost_started) {
 917        return;
 918    }
 919
 920    if (!c->vhost_started) {
 921        int r;
 922
 923        c->vhost_started = 1;
 924        r = cryptodev_vhost_start(vdev, queues);
 925        if (r < 0) {
 926            error_report("unable to start vhost crypto: %d: "
 927                         "falling back on userspace virtio", -r);
 928            c->vhost_started = 0;
 929        }
 930    } else {
 931        cryptodev_vhost_stop(vdev, queues);
 932        c->vhost_started = 0;
 933    }
 934}
 935
 936static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status)
 937{
 938    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 939
 940    virtio_crypto_vhost_status(vcrypto, status);
 941}
 942
 943static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx,
 944                                           bool mask)
 945{
 946    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 947    int queue = virtio_crypto_vq2q(idx);
 948
 949    assert(vcrypto->vhost_started);
 950
 951    cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask);
 952}
 953
 954static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx)
 955{
 956    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 957    int queue = virtio_crypto_vq2q(idx);
 958
 959    assert(vcrypto->vhost_started);
 960
 961    return cryptodev_vhost_virtqueue_pending(vdev, queue, idx);
 962}
 963
 964static void virtio_crypto_class_init(ObjectClass *klass, void *data)
 965{
 966    DeviceClass *dc = DEVICE_CLASS(klass);
 967    VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 968
 969    device_class_set_props(dc, virtio_crypto_properties);
 970    dc->vmsd = &vmstate_virtio_crypto;
 971    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 972    vdc->realize = virtio_crypto_device_realize;
 973    vdc->unrealize = virtio_crypto_device_unrealize;
 974    vdc->get_config = virtio_crypto_get_config;
 975    vdc->get_features = virtio_crypto_get_features;
 976    vdc->reset = virtio_crypto_reset;
 977    vdc->set_status = virtio_crypto_set_status;
 978    vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask;
 979    vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending;
 980}
 981
 982static void virtio_crypto_instance_init(Object *obj)
 983{
 984    VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj);
 985
 986    /*
 987     * The default config_size is sizeof(struct virtio_crypto_config).
 988     * Can be overriden with virtio_crypto_set_config_size.
 989     */
 990    vcrypto->config_size = sizeof(struct virtio_crypto_config);
 991}
 992
 993static const TypeInfo virtio_crypto_info = {
 994    .name = TYPE_VIRTIO_CRYPTO,
 995    .parent = TYPE_VIRTIO_DEVICE,
 996    .instance_size = sizeof(VirtIOCrypto),
 997    .instance_init = virtio_crypto_instance_init,
 998    .class_init = virtio_crypto_class_init,
 999};
1000
1001static void virtio_register_types(void)
1002{
1003    type_register_static(&virtio_crypto_info);
1004}
1005
1006type_init(virtio_register_types)
1007