1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include <linux/vfio.h>
16#include "qapi/error.h"
17#include "qapi/qmp/qdict.h"
18#include "qapi/qmp/qstring.h"
19#include "qemu/error-report.h"
20#include "qemu/cutils.h"
21#include "qemu/option.h"
22#include "qemu/vfio-helpers.h"
23#include "block/block_int.h"
24#include "trace.h"
25
26#include "block/nvme.h"
27
28#define NVME_SQ_ENTRY_BYTES 64
29#define NVME_CQ_ENTRY_BYTES 16
30#define NVME_QUEUE_SIZE 128
31#define NVME_BAR_SIZE 8192
32
33typedef struct {
34 int32_t head, tail;
35 uint8_t *queue;
36 uint64_t iova;
37
38 volatile uint32_t *doorbell;
39} NVMeQueue;
40
41typedef struct {
42 BlockCompletionFunc *cb;
43 void *opaque;
44 int cid;
45 void *prp_list_page;
46 uint64_t prp_list_iova;
47 bool busy;
48} NVMeRequest;
49
50typedef struct {
51 CoQueue free_req_queue;
52 QemuMutex lock;
53
54
55 int index;
56 uint8_t *prp_list_pages;
57
58
59 NVMeQueue sq, cq;
60 int cq_phase;
61 NVMeRequest reqs[NVME_QUEUE_SIZE];
62 bool busy;
63 int need_kick;
64 int inflight;
65} NVMeQueuePair;
66
67
68typedef volatile struct {
69 uint64_t cap;
70 uint32_t vs;
71 uint32_t intms;
72 uint32_t intmc;
73 uint32_t cc;
74 uint32_t reserved0;
75 uint32_t csts;
76 uint32_t nssr;
77 uint32_t aqa;
78 uint64_t asq;
79 uint64_t acq;
80 uint32_t cmbloc;
81 uint32_t cmbsz;
82 uint8_t reserved1[0xec0];
83 uint8_t cmd_set_specfic[0x100];
84 uint32_t doorbells[];
85} QEMU_PACKED NVMeRegs;
86
87QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000);
88
89typedef struct {
90 AioContext *aio_context;
91 QEMUVFIOState *vfio;
92 NVMeRegs *regs;
93
94
95
96
97 NVMeQueuePair **queues;
98 int nr_queues;
99 size_t page_size;
100
101 size_t doorbell_scale;
102 bool write_cache_supported;
103 EventNotifier irq_notifier;
104 uint64_t nsze;
105 int nsid;
106 uint64_t max_transfer;
107 bool plugged;
108
109 CoMutex dma_map_lock;
110 CoQueue dma_flush_queue;
111
112
113 int dma_map_count;
114} BDRVNVMeState;
115
116#define NVME_BLOCK_OPT_DEVICE "device"
117#define NVME_BLOCK_OPT_NAMESPACE "namespace"
118
119static QemuOptsList runtime_opts = {
120 .name = "nvme",
121 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
122 .desc = {
123 {
124 .name = NVME_BLOCK_OPT_DEVICE,
125 .type = QEMU_OPT_STRING,
126 .help = "NVMe PCI device address",
127 },
128 {
129 .name = NVME_BLOCK_OPT_NAMESPACE,
130 .type = QEMU_OPT_NUMBER,
131 .help = "NVMe namespace",
132 },
133 { }
134 },
135};
136
137static void nvme_init_queue(BlockDriverState *bs, NVMeQueue *q,
138 int nentries, int entry_bytes, Error **errp)
139{
140 BDRVNVMeState *s = bs->opaque;
141 size_t bytes;
142 int r;
143
144 bytes = ROUND_UP(nentries * entry_bytes, s->page_size);
145 q->head = q->tail = 0;
146 q->queue = qemu_try_blockalign0(bs, bytes);
147
148 if (!q->queue) {
149 error_setg(errp, "Cannot allocate queue");
150 return;
151 }
152 r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
153 if (r) {
154 error_setg(errp, "Cannot map queue");
155 }
156}
157
158static void nvme_free_queue_pair(BlockDriverState *bs, NVMeQueuePair *q)
159{
160 qemu_vfree(q->prp_list_pages);
161 qemu_vfree(q->sq.queue);
162 qemu_vfree(q->cq.queue);
163 qemu_mutex_destroy(&q->lock);
164 g_free(q);
165}
166
167static void nvme_free_req_queue_cb(void *opaque)
168{
169 NVMeQueuePair *q = opaque;
170
171 qemu_mutex_lock(&q->lock);
172 while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
173
174 }
175 qemu_mutex_unlock(&q->lock);
176}
177
178static NVMeQueuePair *nvme_create_queue_pair(BlockDriverState *bs,
179 int idx, int size,
180 Error **errp)
181{
182 int i, r;
183 BDRVNVMeState *s = bs->opaque;
184 Error *local_err = NULL;
185 NVMeQueuePair *q = g_new0(NVMeQueuePair, 1);
186 uint64_t prp_list_iova;
187
188 qemu_mutex_init(&q->lock);
189 q->index = idx;
190 qemu_co_queue_init(&q->free_req_queue);
191 q->prp_list_pages = qemu_blockalign0(bs, s->page_size * NVME_QUEUE_SIZE);
192 r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages,
193 s->page_size * NVME_QUEUE_SIZE,
194 false, &prp_list_iova);
195 if (r) {
196 goto fail;
197 }
198 for (i = 0; i < NVME_QUEUE_SIZE; i++) {
199 NVMeRequest *req = &q->reqs[i];
200 req->cid = i + 1;
201 req->prp_list_page = q->prp_list_pages + i * s->page_size;
202 req->prp_list_iova = prp_list_iova + i * s->page_size;
203 }
204 nvme_init_queue(bs, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err);
205 if (local_err) {
206 error_propagate(errp, local_err);
207 goto fail;
208 }
209 q->sq.doorbell = &s->regs->doorbells[idx * 2 * s->doorbell_scale];
210
211 nvme_init_queue(bs, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err);
212 if (local_err) {
213 error_propagate(errp, local_err);
214 goto fail;
215 }
216 q->cq.doorbell = &s->regs->doorbells[idx * 2 * s->doorbell_scale + 1];
217
218 return q;
219fail:
220 nvme_free_queue_pair(bs, q);
221 return NULL;
222}
223
224
225static void nvme_kick(BDRVNVMeState *s, NVMeQueuePair *q)
226{
227 if (s->plugged || !q->need_kick) {
228 return;
229 }
230 trace_nvme_kick(s, q->index);
231 assert(!(q->sq.tail & 0xFF00));
232
233 smp_wmb();
234 *q->sq.doorbell = cpu_to_le32(q->sq.tail);
235 q->inflight += q->need_kick;
236 q->need_kick = 0;
237}
238
239
240
241
242
243static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
244{
245 int i;
246 NVMeRequest *req = NULL;
247
248 qemu_mutex_lock(&q->lock);
249 while (q->inflight + q->need_kick > NVME_QUEUE_SIZE - 2) {
250
251
252 if (qemu_in_coroutine()) {
253 trace_nvme_free_req_queue_wait(q);
254 qemu_co_queue_wait(&q->free_req_queue, &q->lock);
255 } else {
256 qemu_mutex_unlock(&q->lock);
257 return NULL;
258 }
259 }
260 for (i = 0; i < NVME_QUEUE_SIZE; i++) {
261 if (!q->reqs[i].busy) {
262 q->reqs[i].busy = true;
263 req = &q->reqs[i];
264 break;
265 }
266 }
267
268
269 assert(req);
270 qemu_mutex_unlock(&q->lock);
271 return req;
272}
273
274static inline int nvme_translate_error(const NvmeCqe *c)
275{
276 uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
277 if (status) {
278 trace_nvme_error(le32_to_cpu(c->result),
279 le16_to_cpu(c->sq_head),
280 le16_to_cpu(c->sq_id),
281 le16_to_cpu(c->cid),
282 le16_to_cpu(status));
283 }
284 switch (status) {
285 case 0:
286 return 0;
287 case 1:
288 return -ENOSYS;
289 case 2:
290 return -EINVAL;
291 default:
292 return -EIO;
293 }
294}
295
296
297static bool nvme_process_completion(BDRVNVMeState *s, NVMeQueuePair *q)
298{
299 bool progress = false;
300 NVMeRequest *preq;
301 NVMeRequest req;
302 NvmeCqe *c;
303
304 trace_nvme_process_completion(s, q->index, q->inflight);
305 if (q->busy || s->plugged) {
306 trace_nvme_process_completion_queue_busy(s, q->index);
307 return false;
308 }
309 q->busy = true;
310 assert(q->inflight >= 0);
311 while (q->inflight) {
312 int16_t cid;
313 c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
314 if (!c->cid || (le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
315 break;
316 }
317 q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
318 if (!q->cq.head) {
319 q->cq_phase = !q->cq_phase;
320 }
321 cid = le16_to_cpu(c->cid);
322 if (cid == 0 || cid > NVME_QUEUE_SIZE) {
323 fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n",
324 cid);
325 continue;
326 }
327 assert(cid <= NVME_QUEUE_SIZE);
328 trace_nvme_complete_command(s, q->index, cid);
329 preq = &q->reqs[cid - 1];
330 req = *preq;
331 assert(req.cid == cid);
332 assert(req.cb);
333 preq->busy = false;
334 preq->cb = preq->opaque = NULL;
335 qemu_mutex_unlock(&q->lock);
336 req.cb(req.opaque, nvme_translate_error(c));
337 qemu_mutex_lock(&q->lock);
338 c->cid = cpu_to_le16(0);
339 q->inflight--;
340
341 c->status = cpu_to_le16(le16_to_cpu(c->status) ^ 0x1);
342 progress = true;
343 }
344 if (progress) {
345
346 smp_mb_release();
347 *q->cq.doorbell = cpu_to_le32(q->cq.head);
348 if (!qemu_co_queue_empty(&q->free_req_queue)) {
349 aio_bh_schedule_oneshot(s->aio_context, nvme_free_req_queue_cb, q);
350 }
351 }
352 q->busy = false;
353 return progress;
354}
355
356static void nvme_trace_command(const NvmeCmd *cmd)
357{
358 int i;
359
360 for (i = 0; i < 8; ++i) {
361 uint8_t *cmdp = (uint8_t *)cmd + i * 8;
362 trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
363 cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
364 }
365}
366
367static void nvme_submit_command(BDRVNVMeState *s, NVMeQueuePair *q,
368 NVMeRequest *req,
369 NvmeCmd *cmd, BlockCompletionFunc cb,
370 void *opaque)
371{
372 assert(!req->cb);
373 req->cb = cb;
374 req->opaque = opaque;
375 cmd->cid = cpu_to_le32(req->cid);
376
377 trace_nvme_submit_command(s, q->index, req->cid);
378 nvme_trace_command(cmd);
379 qemu_mutex_lock(&q->lock);
380 memcpy((uint8_t *)q->sq.queue +
381 q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
382 q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
383 q->need_kick++;
384 nvme_kick(s, q);
385 nvme_process_completion(s, q);
386 qemu_mutex_unlock(&q->lock);
387}
388
389static void nvme_cmd_sync_cb(void *opaque, int ret)
390{
391 int *pret = opaque;
392 *pret = ret;
393 aio_wait_kick();
394}
395
396static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
397 NvmeCmd *cmd)
398{
399 NVMeRequest *req;
400 BDRVNVMeState *s = bs->opaque;
401 int ret = -EINPROGRESS;
402 req = nvme_get_free_req(q);
403 if (!req) {
404 return -EBUSY;
405 }
406 nvme_submit_command(s, q, req, cmd, nvme_cmd_sync_cb, &ret);
407
408 BDRV_POLL_WHILE(bs, ret == -EINPROGRESS);
409 return ret;
410}
411
412static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
413{
414 BDRVNVMeState *s = bs->opaque;
415 NvmeIdCtrl *idctrl;
416 NvmeIdNs *idns;
417 uint8_t *resp;
418 int r;
419 uint64_t iova;
420 NvmeCmd cmd = {
421 .opcode = NVME_ADM_CMD_IDENTIFY,
422 .cdw10 = cpu_to_le32(0x1),
423 };
424
425 resp = qemu_try_blockalign0(bs, sizeof(NvmeIdCtrl));
426 if (!resp) {
427 error_setg(errp, "Cannot allocate buffer for identify response");
428 goto out;
429 }
430 idctrl = (NvmeIdCtrl *)resp;
431 idns = (NvmeIdNs *)resp;
432 r = qemu_vfio_dma_map(s->vfio, resp, sizeof(NvmeIdCtrl), true, &iova);
433 if (r) {
434 error_setg(errp, "Cannot map buffer for DMA");
435 goto out;
436 }
437 cmd.prp1 = cpu_to_le64(iova);
438
439 if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
440 error_setg(errp, "Failed to identify controller");
441 goto out;
442 }
443
444 if (le32_to_cpu(idctrl->nn) < namespace) {
445 error_setg(errp, "Invalid namespace");
446 goto out;
447 }
448 s->write_cache_supported = le32_to_cpu(idctrl->vwc) & 0x1;
449 s->max_transfer = (idctrl->mdts ? 1 << idctrl->mdts : 0) * s->page_size;
450
451
452 s->max_transfer = MIN_NON_ZERO(s->max_transfer,
453 s->page_size / sizeof(uint64_t) * s->page_size);
454
455 memset(resp, 0, 4096);
456
457 cmd.cdw10 = 0;
458 cmd.nsid = cpu_to_le32(namespace);
459 if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
460 error_setg(errp, "Failed to identify namespace");
461 goto out;
462 }
463
464 s->nsze = le64_to_cpu(idns->nsze);
465
466out:
467 qemu_vfio_dma_unmap(s->vfio, resp);
468 qemu_vfree(resp);
469}
470
471static bool nvme_poll_queues(BDRVNVMeState *s)
472{
473 bool progress = false;
474 int i;
475
476 for (i = 0; i < s->nr_queues; i++) {
477 NVMeQueuePair *q = s->queues[i];
478 qemu_mutex_lock(&q->lock);
479 while (nvme_process_completion(s, q)) {
480
481 progress = true;
482 }
483 qemu_mutex_unlock(&q->lock);
484 }
485 return progress;
486}
487
488static void nvme_handle_event(EventNotifier *n)
489{
490 BDRVNVMeState *s = container_of(n, BDRVNVMeState, irq_notifier);
491
492 trace_nvme_handle_event(s);
493 event_notifier_test_and_clear(n);
494 nvme_poll_queues(s);
495}
496
497static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
498{
499 BDRVNVMeState *s = bs->opaque;
500 int n = s->nr_queues;
501 NVMeQueuePair *q;
502 NvmeCmd cmd;
503 int queue_size = NVME_QUEUE_SIZE;
504
505 q = nvme_create_queue_pair(bs, n, queue_size, errp);
506 if (!q) {
507 return false;
508 }
509 cmd = (NvmeCmd) {
510 .opcode = NVME_ADM_CMD_CREATE_CQ,
511 .prp1 = cpu_to_le64(q->cq.iova),
512 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
513 .cdw11 = cpu_to_le32(0x3),
514 };
515 if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
516 error_setg(errp, "Failed to create io queue [%d]", n);
517 nvme_free_queue_pair(bs, q);
518 return false;
519 }
520 cmd = (NvmeCmd) {
521 .opcode = NVME_ADM_CMD_CREATE_SQ,
522 .prp1 = cpu_to_le64(q->sq.iova),
523 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
524 .cdw11 = cpu_to_le32(0x1 | (n << 16)),
525 };
526 if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
527 error_setg(errp, "Failed to create io queue [%d]", n);
528 nvme_free_queue_pair(bs, q);
529 return false;
530 }
531 s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
532 s->queues[n] = q;
533 s->nr_queues++;
534 return true;
535}
536
537static bool nvme_poll_cb(void *opaque)
538{
539 EventNotifier *e = opaque;
540 BDRVNVMeState *s = container_of(e, BDRVNVMeState, irq_notifier);
541 bool progress = false;
542
543 trace_nvme_poll_cb(s);
544 progress = nvme_poll_queues(s);
545 return progress;
546}
547
548static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
549 Error **errp)
550{
551 BDRVNVMeState *s = bs->opaque;
552 int ret;
553 uint64_t cap;
554 uint64_t timeout_ms;
555 uint64_t deadline, now;
556 Error *local_err = NULL;
557
558 qemu_co_mutex_init(&s->dma_map_lock);
559 qemu_co_queue_init(&s->dma_flush_queue);
560 s->nsid = namespace;
561 s->aio_context = bdrv_get_aio_context(bs);
562 ret = event_notifier_init(&s->irq_notifier, 0);
563 if (ret) {
564 error_setg(errp, "Failed to init event notifier");
565 return ret;
566 }
567
568 s->vfio = qemu_vfio_open_pci(device, errp);
569 if (!s->vfio) {
570 ret = -EINVAL;
571 goto out;
572 }
573
574 s->regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, NVME_BAR_SIZE, errp);
575 if (!s->regs) {
576 ret = -EINVAL;
577 goto out;
578 }
579
580
581
582
583 cap = le64_to_cpu(s->regs->cap);
584 if (!(cap & (1ULL << 37))) {
585 error_setg(errp, "Device doesn't support NVMe command set");
586 ret = -EINVAL;
587 goto out;
588 }
589
590 s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF)));
591 s->doorbell_scale = (4 << (((cap >> 32) & 0xF))) / sizeof(uint32_t);
592 bs->bl.opt_mem_alignment = s->page_size;
593 timeout_ms = MIN(500 * ((cap >> 24) & 0xFF), 30000);
594
595
596 s->regs->cc = cpu_to_le32(le32_to_cpu(s->regs->cc) & 0xFE);
597
598 deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * 1000000ULL;
599 while (le32_to_cpu(s->regs->csts) & 0x1) {
600 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
601 error_setg(errp, "Timeout while waiting for device to reset (%"
602 PRId64 " ms)",
603 timeout_ms);
604 ret = -ETIMEDOUT;
605 goto out;
606 }
607 }
608
609
610 s->queues = g_new(NVMeQueuePair *, 1);
611 s->nr_queues = 1;
612 s->queues[0] = nvme_create_queue_pair(bs, 0, NVME_QUEUE_SIZE, errp);
613 if (!s->queues[0]) {
614 ret = -EINVAL;
615 goto out;
616 }
617 QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
618 s->regs->aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE);
619 s->regs->asq = cpu_to_le64(s->queues[0]->sq.iova);
620 s->regs->acq = cpu_to_le64(s->queues[0]->cq.iova);
621
622
623 s->regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << 20) |
624 (ctz32(NVME_SQ_ENTRY_BYTES) << 16) |
625 0x1);
626
627 now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
628 deadline = now + timeout_ms * 1000000;
629 while (!(le32_to_cpu(s->regs->csts) & 0x1)) {
630 if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
631 error_setg(errp, "Timeout while waiting for device to start (%"
632 PRId64 " ms)",
633 timeout_ms);
634 ret = -ETIMEDOUT;
635 goto out;
636 }
637 }
638
639 ret = qemu_vfio_pci_init_irq(s->vfio, &s->irq_notifier,
640 VFIO_PCI_MSIX_IRQ_INDEX, errp);
641 if (ret) {
642 goto out;
643 }
644 aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
645 false, nvme_handle_event, nvme_poll_cb);
646
647 nvme_identify(bs, namespace, &local_err);
648 if (local_err) {
649 error_propagate(errp, local_err);
650 ret = -EIO;
651 goto out;
652 }
653
654
655 if (!nvme_add_io_queue(bs, errp)) {
656 ret = -EIO;
657 }
658out:
659
660 return ret;
661}
662
663
664
665
666
667
668
669
670static void nvme_parse_filename(const char *filename, QDict *options,
671 Error **errp)
672{
673 int pref = strlen("nvme://");
674
675 if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
676 const char *tmp = filename + pref;
677 char *device;
678 const char *namespace;
679 unsigned long ns;
680 const char *slash = strchr(tmp, '/');
681 if (!slash) {
682 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
683 return;
684 }
685 device = g_strndup(tmp, slash - tmp);
686 qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
687 g_free(device);
688 namespace = slash + 1;
689 if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
690 error_setg(errp, "Invalid namespace '%s', positive number expected",
691 namespace);
692 return;
693 }
694 qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
695 *namespace ? namespace : "1");
696 }
697}
698
699static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
700 Error **errp)
701{
702 int ret;
703 BDRVNVMeState *s = bs->opaque;
704 NvmeCmd cmd = {
705 .opcode = NVME_ADM_CMD_SET_FEATURES,
706 .nsid = cpu_to_le32(s->nsid),
707 .cdw10 = cpu_to_le32(0x06),
708 .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
709 };
710
711 ret = nvme_cmd_sync(bs, s->queues[0], &cmd);
712 if (ret) {
713 error_setg(errp, "Failed to configure NVMe write cache");
714 }
715 return ret;
716}
717
718static void nvme_close(BlockDriverState *bs)
719{
720 int i;
721 BDRVNVMeState *s = bs->opaque;
722
723 for (i = 0; i < s->nr_queues; ++i) {
724 nvme_free_queue_pair(bs, s->queues[i]);
725 }
726 g_free(s->queues);
727 aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
728 false, NULL, NULL);
729 event_notifier_cleanup(&s->irq_notifier);
730 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE);
731 qemu_vfio_close(s->vfio);
732}
733
734static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
735 Error **errp)
736{
737 const char *device;
738 QemuOpts *opts;
739 int namespace;
740 int ret;
741 BDRVNVMeState *s = bs->opaque;
742
743 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
744 qemu_opts_absorb_qdict(opts, options, &error_abort);
745 device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
746 if (!device) {
747 error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
748 qemu_opts_del(opts);
749 return -EINVAL;
750 }
751
752 namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
753 ret = nvme_init(bs, device, namespace, errp);
754 qemu_opts_del(opts);
755 if (ret) {
756 goto fail;
757 }
758 if (flags & BDRV_O_NOCACHE) {
759 if (!s->write_cache_supported) {
760 error_setg(errp,
761 "NVMe controller doesn't support write cache configuration");
762 ret = -EINVAL;
763 } else {
764 ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
765 errp);
766 }
767 if (ret) {
768 goto fail;
769 }
770 }
771 bs->supported_write_flags = BDRV_REQ_FUA;
772 return 0;
773fail:
774 nvme_close(bs);
775 return ret;
776}
777
778static int64_t nvme_getlength(BlockDriverState *bs)
779{
780 BDRVNVMeState *s = bs->opaque;
781
782 return s->nsze << BDRV_SECTOR_BITS;
783}
784
785
786static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
787 QEMUIOVector *qiov)
788{
789 int r = 0;
790 BDRVNVMeState *s = bs->opaque;
791
792 s->dma_map_count -= qiov->size;
793 if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
794 r = qemu_vfio_dma_reset_temporary(s->vfio);
795 if (!r) {
796 qemu_co_queue_restart_all(&s->dma_flush_queue);
797 }
798 }
799 return r;
800}
801
802
803static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
804 NVMeRequest *req, QEMUIOVector *qiov)
805{
806 BDRVNVMeState *s = bs->opaque;
807 uint64_t *pagelist = req->prp_list_page;
808 int i, j, r;
809 int entries = 0;
810
811 assert(qiov->size);
812 assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
813 assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
814 for (i = 0; i < qiov->niov; ++i) {
815 bool retry = true;
816 uint64_t iova;
817try_map:
818 r = qemu_vfio_dma_map(s->vfio,
819 qiov->iov[i].iov_base,
820 qiov->iov[i].iov_len,
821 true, &iova);
822 if (r == -ENOMEM && retry) {
823 retry = false;
824 trace_nvme_dma_flush_queue_wait(s);
825 if (s->dma_map_count) {
826 trace_nvme_dma_map_flush(s);
827 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
828 } else {
829 r = qemu_vfio_dma_reset_temporary(s->vfio);
830 if (r) {
831 goto fail;
832 }
833 }
834 goto try_map;
835 }
836 if (r) {
837 goto fail;
838 }
839
840 for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
841 pagelist[entries++] = iova + j * s->page_size;
842 }
843 trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
844 qiov->iov[i].iov_len / s->page_size);
845 }
846
847 s->dma_map_count += qiov->size;
848
849 assert(entries <= s->page_size / sizeof(uint64_t));
850 switch (entries) {
851 case 0:
852 abort();
853 case 1:
854 cmd->prp1 = cpu_to_le64(pagelist[0]);
855 cmd->prp2 = 0;
856 break;
857 case 2:
858 cmd->prp1 = cpu_to_le64(pagelist[0]);
859 cmd->prp2 = cpu_to_le64(pagelist[1]);;
860 break;
861 default:
862 cmd->prp1 = cpu_to_le64(pagelist[0]);
863 cmd->prp2 = cpu_to_le64(req->prp_list_iova);
864 for (i = 0; i < entries - 1; ++i) {
865 pagelist[i] = cpu_to_le64(pagelist[i + 1]);
866 }
867 pagelist[entries - 1] = 0;
868 break;
869 }
870 trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
871 for (i = 0; i < entries; ++i) {
872 trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
873 }
874 return 0;
875fail:
876
877
878
879
880
881 return r;
882}
883
884typedef struct {
885 Coroutine *co;
886 int ret;
887 AioContext *ctx;
888} NVMeCoData;
889
890static void nvme_rw_cb_bh(void *opaque)
891{
892 NVMeCoData *data = opaque;
893 qemu_coroutine_enter(data->co);
894}
895
896static void nvme_rw_cb(void *opaque, int ret)
897{
898 NVMeCoData *data = opaque;
899 data->ret = ret;
900 if (!data->co) {
901
902 return;
903 }
904 aio_bh_schedule_oneshot(data->ctx, nvme_rw_cb_bh, data);
905}
906
907static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
908 uint64_t offset, uint64_t bytes,
909 QEMUIOVector *qiov,
910 bool is_write,
911 int flags)
912{
913 int r;
914 BDRVNVMeState *s = bs->opaque;
915 NVMeQueuePair *ioq = s->queues[1];
916 NVMeRequest *req;
917 uint32_t cdw12 = (((bytes >> BDRV_SECTOR_BITS) - 1) & 0xFFFF) |
918 (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
919 NvmeCmd cmd = {
920 .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
921 .nsid = cpu_to_le32(s->nsid),
922 .cdw10 = cpu_to_le32((offset >> BDRV_SECTOR_BITS) & 0xFFFFFFFF),
923 .cdw11 = cpu_to_le32(((offset >> BDRV_SECTOR_BITS) >> 32) & 0xFFFFFFFF),
924 .cdw12 = cpu_to_le32(cdw12),
925 };
926 NVMeCoData data = {
927 .ctx = bdrv_get_aio_context(bs),
928 .ret = -EINPROGRESS,
929 };
930
931 trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
932 assert(s->nr_queues > 1);
933 req = nvme_get_free_req(ioq);
934 assert(req);
935
936 qemu_co_mutex_lock(&s->dma_map_lock);
937 r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
938 qemu_co_mutex_unlock(&s->dma_map_lock);
939 if (r) {
940 req->busy = false;
941 return r;
942 }
943 nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
944
945 data.co = qemu_coroutine_self();
946 while (data.ret == -EINPROGRESS) {
947 qemu_coroutine_yield();
948 }
949
950 qemu_co_mutex_lock(&s->dma_map_lock);
951 r = nvme_cmd_unmap_qiov(bs, qiov);
952 qemu_co_mutex_unlock(&s->dma_map_lock);
953 if (r) {
954 return r;
955 }
956
957 trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
958 return data.ret;
959}
960
961static inline bool nvme_qiov_aligned(BlockDriverState *bs,
962 const QEMUIOVector *qiov)
963{
964 int i;
965 BDRVNVMeState *s = bs->opaque;
966
967 for (i = 0; i < qiov->niov; ++i) {
968 if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) ||
969 !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) {
970 trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
971 qiov->iov[i].iov_len, s->page_size);
972 return false;
973 }
974 }
975 return true;
976}
977
978static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
979 QEMUIOVector *qiov, bool is_write, int flags)
980{
981 BDRVNVMeState *s = bs->opaque;
982 int r;
983 uint8_t *buf = NULL;
984 QEMUIOVector local_qiov;
985
986 assert(QEMU_IS_ALIGNED(offset, s->page_size));
987 assert(QEMU_IS_ALIGNED(bytes, s->page_size));
988 assert(bytes <= s->max_transfer);
989 if (nvme_qiov_aligned(bs, qiov)) {
990 return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
991 }
992 trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
993 buf = qemu_try_blockalign(bs, bytes);
994
995 if (!buf) {
996 return -ENOMEM;
997 }
998 qemu_iovec_init(&local_qiov, 1);
999 if (is_write) {
1000 qemu_iovec_to_buf(qiov, 0, buf, bytes);
1001 }
1002 qemu_iovec_add(&local_qiov, buf, bytes);
1003 r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1004 qemu_iovec_destroy(&local_qiov);
1005 if (!r && !is_write) {
1006 qemu_iovec_from_buf(qiov, 0, buf, bytes);
1007 }
1008 qemu_vfree(buf);
1009 return r;
1010}
1011
1012static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1013 uint64_t offset, uint64_t bytes,
1014 QEMUIOVector *qiov, int flags)
1015{
1016 return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1017}
1018
1019static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1020 uint64_t offset, uint64_t bytes,
1021 QEMUIOVector *qiov, int flags)
1022{
1023 return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1024}
1025
1026static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1027{
1028 BDRVNVMeState *s = bs->opaque;
1029 NVMeQueuePair *ioq = s->queues[1];
1030 NVMeRequest *req;
1031 NvmeCmd cmd = {
1032 .opcode = NVME_CMD_FLUSH,
1033 .nsid = cpu_to_le32(s->nsid),
1034 };
1035 NVMeCoData data = {
1036 .ctx = bdrv_get_aio_context(bs),
1037 .ret = -EINPROGRESS,
1038 };
1039
1040 assert(s->nr_queues > 1);
1041 req = nvme_get_free_req(ioq);
1042 assert(req);
1043 nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
1044
1045 data.co = qemu_coroutine_self();
1046 if (data.ret == -EINPROGRESS) {
1047 qemu_coroutine_yield();
1048 }
1049
1050 return data.ret;
1051}
1052
1053
1054static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1055 BlockReopenQueue *queue, Error **errp)
1056{
1057 return 0;
1058}
1059
1060static void nvme_refresh_filename(BlockDriverState *bs, QDict *opts)
1061{
1062 qdict_del(opts, "filename");
1063
1064 if (!qdict_size(opts)) {
1065 snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s://",
1066 bs->drv->format_name);
1067 }
1068
1069 qdict_put_str(opts, "driver", bs->drv->format_name);
1070 bs->full_open_options = qobject_ref(opts);
1071}
1072
1073static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1074{
1075 BDRVNVMeState *s = bs->opaque;
1076
1077 bs->bl.opt_mem_alignment = s->page_size;
1078 bs->bl.request_alignment = s->page_size;
1079 bs->bl.max_transfer = s->max_transfer;
1080}
1081
1082static void nvme_detach_aio_context(BlockDriverState *bs)
1083{
1084 BDRVNVMeState *s = bs->opaque;
1085
1086 aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
1087 false, NULL, NULL);
1088}
1089
1090static void nvme_attach_aio_context(BlockDriverState *bs,
1091 AioContext *new_context)
1092{
1093 BDRVNVMeState *s = bs->opaque;
1094
1095 s->aio_context = new_context;
1096 aio_set_event_notifier(new_context, &s->irq_notifier,
1097 false, nvme_handle_event, nvme_poll_cb);
1098}
1099
1100static void nvme_aio_plug(BlockDriverState *bs)
1101{
1102 BDRVNVMeState *s = bs->opaque;
1103 assert(!s->plugged);
1104 s->plugged = true;
1105}
1106
1107static void nvme_aio_unplug(BlockDriverState *bs)
1108{
1109 int i;
1110 BDRVNVMeState *s = bs->opaque;
1111 assert(s->plugged);
1112 s->plugged = false;
1113 for (i = 1; i < s->nr_queues; i++) {
1114 NVMeQueuePair *q = s->queues[i];
1115 qemu_mutex_lock(&q->lock);
1116 nvme_kick(s, q);
1117 nvme_process_completion(s, q);
1118 qemu_mutex_unlock(&q->lock);
1119 }
1120}
1121
1122static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1123{
1124 int ret;
1125 BDRVNVMeState *s = bs->opaque;
1126
1127 ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1128 if (ret) {
1129
1130
1131
1132 error_report("nvme_register_buf failed: %s", strerror(-ret));
1133 }
1134}
1135
1136static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1137{
1138 BDRVNVMeState *s = bs->opaque;
1139
1140 qemu_vfio_dma_unmap(s->vfio, host);
1141}
1142
1143static BlockDriver bdrv_nvme = {
1144 .format_name = "nvme",
1145 .protocol_name = "nvme",
1146 .instance_size = sizeof(BDRVNVMeState),
1147
1148 .bdrv_parse_filename = nvme_parse_filename,
1149 .bdrv_file_open = nvme_file_open,
1150 .bdrv_close = nvme_close,
1151 .bdrv_getlength = nvme_getlength,
1152
1153 .bdrv_co_preadv = nvme_co_preadv,
1154 .bdrv_co_pwritev = nvme_co_pwritev,
1155 .bdrv_co_flush_to_disk = nvme_co_flush,
1156 .bdrv_reopen_prepare = nvme_reopen_prepare,
1157
1158 .bdrv_refresh_filename = nvme_refresh_filename,
1159 .bdrv_refresh_limits = nvme_refresh_limits,
1160
1161 .bdrv_detach_aio_context = nvme_detach_aio_context,
1162 .bdrv_attach_aio_context = nvme_attach_aio_context,
1163
1164 .bdrv_io_plug = nvme_aio_plug,
1165 .bdrv_io_unplug = nvme_aio_unplug,
1166
1167 .bdrv_register_buf = nvme_register_buf,
1168 .bdrv_unregister_buf = nvme_unregister_buf,
1169};
1170
1171static void bdrv_nvme_init(void)
1172{
1173 bdrv_register(&bdrv_nvme);
1174}
1175
1176block_init(bdrv_nvme_init);
1177