1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include "qemu/osdep.h"
24#include <hw/block/block.h>
25#include <hw/hw.h>
26#include <hw/pci/msix.h>
27#include <hw/pci/pci.h>
28#include "sysemu/sysemu.h"
29#include "qapi/error.h"
30#include "qapi/visitor.h"
31#include "sysemu/block-backend.h"
32
33#include "nvme.h"
34
35static void nvme_process_sq(void *opaque);
36
37static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
38{
39 return sqid < n->num_queues && n->sq[sqid] != NULL ? 0 : -1;
40}
41
42static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
43{
44 return cqid < n->num_queues && n->cq[cqid] != NULL ? 0 : -1;
45}
46
47static void nvme_inc_cq_tail(NvmeCQueue *cq)
48{
49 cq->tail++;
50 if (cq->tail >= cq->size) {
51 cq->tail = 0;
52 cq->phase = !cq->phase;
53 }
54}
55
56static void nvme_inc_sq_head(NvmeSQueue *sq)
57{
58 sq->head = (sq->head + 1) % sq->size;
59}
60
61static uint8_t nvme_cq_full(NvmeCQueue *cq)
62{
63 return (cq->tail + 1) % cq->size == cq->head;
64}
65
66static uint8_t nvme_sq_empty(NvmeSQueue *sq)
67{
68 return sq->head == sq->tail;
69}
70
71static void nvme_isr_notify(NvmeCtrl *n, NvmeCQueue *cq)
72{
73 if (cq->irq_enabled) {
74 if (msix_enabled(&(n->parent_obj))) {
75 msix_notify(&(n->parent_obj), cq->vector);
76 } else {
77 pci_irq_pulse(&n->parent_obj);
78 }
79 }
80}
81
82static uint16_t nvme_map_prp(QEMUSGList *qsg, uint64_t prp1, uint64_t prp2,
83 uint32_t len, NvmeCtrl *n)
84{
85 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
86 trans_len = MIN(len, trans_len);
87 int num_prps = (len >> n->page_bits) + 1;
88
89 if (!prp1) {
90 return NVME_INVALID_FIELD | NVME_DNR;
91 }
92
93 pci_dma_sglist_init(qsg, &n->parent_obj, num_prps);
94 qemu_sglist_add(qsg, prp1, trans_len);
95 len -= trans_len;
96 if (len) {
97 if (!prp2) {
98 goto unmap;
99 }
100 if (len > n->page_size) {
101 uint64_t prp_list[n->max_prp_ents];
102 uint32_t nents, prp_trans;
103 int i = 0;
104
105 nents = (len + n->page_size - 1) >> n->page_bits;
106 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
107 pci_dma_read(&n->parent_obj, prp2, (void *)prp_list, prp_trans);
108 while (len != 0) {
109 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
110
111 if (i == n->max_prp_ents - 1 && len > n->page_size) {
112 if (!prp_ent || prp_ent & (n->page_size - 1)) {
113 goto unmap;
114 }
115
116 i = 0;
117 nents = (len + n->page_size - 1) >> n->page_bits;
118 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
119 pci_dma_read(&n->parent_obj, prp_ent, (void *)prp_list,
120 prp_trans);
121 prp_ent = le64_to_cpu(prp_list[i]);
122 }
123
124 if (!prp_ent || prp_ent & (n->page_size - 1)) {
125 goto unmap;
126 }
127
128 trans_len = MIN(len, n->page_size);
129 qemu_sglist_add(qsg, prp_ent, trans_len);
130 len -= trans_len;
131 i++;
132 }
133 } else {
134 if (prp2 & (n->page_size - 1)) {
135 goto unmap;
136 }
137 qemu_sglist_add(qsg, prp2, len);
138 }
139 }
140 return NVME_SUCCESS;
141
142 unmap:
143 qemu_sglist_destroy(qsg);
144 return NVME_INVALID_FIELD | NVME_DNR;
145}
146
147static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
148 uint64_t prp1, uint64_t prp2)
149{
150 QEMUSGList qsg;
151
152 if (nvme_map_prp(&qsg, prp1, prp2, len, n)) {
153 return NVME_INVALID_FIELD | NVME_DNR;
154 }
155 if (dma_buf_read(ptr, len, &qsg)) {
156 qemu_sglist_destroy(&qsg);
157 return NVME_INVALID_FIELD | NVME_DNR;
158 }
159 qemu_sglist_destroy(&qsg);
160 return NVME_SUCCESS;
161}
162
163static void nvme_post_cqes(void *opaque)
164{
165 NvmeCQueue *cq = opaque;
166 NvmeCtrl *n = cq->ctrl;
167 NvmeRequest *req, *next;
168
169 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
170 NvmeSQueue *sq;
171 hwaddr addr;
172
173 if (nvme_cq_full(cq)) {
174 break;
175 }
176
177 QTAILQ_REMOVE(&cq->req_list, req, entry);
178 sq = req->sq;
179 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
180 req->cqe.sq_id = cpu_to_le16(sq->sqid);
181 req->cqe.sq_head = cpu_to_le16(sq->head);
182 addr = cq->dma_addr + cq->tail * n->cqe_size;
183 nvme_inc_cq_tail(cq);
184 pci_dma_write(&n->parent_obj, addr, (void *)&req->cqe,
185 sizeof(req->cqe));
186 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
187 }
188 nvme_isr_notify(n, cq);
189}
190
191static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
192{
193 assert(cq->cqid == req->sq->cqid);
194 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
195 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
196 timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
197}
198
199static void nvme_rw_cb(void *opaque, int ret)
200{
201 NvmeRequest *req = opaque;
202 NvmeSQueue *sq = req->sq;
203 NvmeCtrl *n = sq->ctrl;
204 NvmeCQueue *cq = n->cq[sq->cqid];
205
206 if (!ret) {
207 block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
208 req->status = NVME_SUCCESS;
209 } else {
210 block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
211 req->status = NVME_INTERNAL_DEV_ERROR;
212 }
213 if (req->has_sg) {
214 qemu_sglist_destroy(&req->qsg);
215 }
216 nvme_enqueue_req_completion(cq, req);
217}
218
219static uint16_t nvme_flush(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
220 NvmeRequest *req)
221{
222 req->has_sg = false;
223 block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
224 BLOCK_ACCT_FLUSH);
225 req->aiocb = blk_aio_flush(n->conf.blk, nvme_rw_cb, req);
226
227 return NVME_NO_COMPLETE;
228}
229
230static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
231 NvmeRequest *req)
232{
233 NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
234 uint32_t nlb = le32_to_cpu(rw->nlb) + 1;
235 uint64_t slba = le64_to_cpu(rw->slba);
236 uint64_t prp1 = le64_to_cpu(rw->prp1);
237 uint64_t prp2 = le64_to_cpu(rw->prp2);
238
239 uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
240 uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
241 uint64_t data_size = (uint64_t)nlb << data_shift;
242 uint64_t aio_slba = slba << (data_shift - BDRV_SECTOR_BITS);
243 int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
244 enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
245
246 if ((slba + nlb) > ns->id_ns.nsze) {
247 block_acct_invalid(blk_get_stats(n->conf.blk), acct);
248 return NVME_LBA_RANGE | NVME_DNR;
249 }
250
251 if (nvme_map_prp(&req->qsg, prp1, prp2, data_size, n)) {
252 block_acct_invalid(blk_get_stats(n->conf.blk), acct);
253 return NVME_INVALID_FIELD | NVME_DNR;
254 }
255
256 assert((nlb << data_shift) == req->qsg.size);
257
258 req->has_sg = true;
259 dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
260 req->aiocb = is_write ?
261 dma_blk_write(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req) :
262 dma_blk_read(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req);
263
264 return NVME_NO_COMPLETE;
265}
266
267static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
268{
269 NvmeNamespace *ns;
270 uint32_t nsid = le32_to_cpu(cmd->nsid);
271
272 if (nsid == 0 || nsid > n->num_namespaces) {
273 return NVME_INVALID_NSID | NVME_DNR;
274 }
275
276 ns = &n->namespaces[nsid - 1];
277 switch (cmd->opcode) {
278 case NVME_CMD_FLUSH:
279 return nvme_flush(n, ns, cmd, req);
280 case NVME_CMD_WRITE:
281 case NVME_CMD_READ:
282 return nvme_rw(n, ns, cmd, req);
283 default:
284 return NVME_INVALID_OPCODE | NVME_DNR;
285 }
286}
287
288static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
289{
290 n->sq[sq->sqid] = NULL;
291 timer_del(sq->timer);
292 timer_free(sq->timer);
293 g_free(sq->io_req);
294 if (sq->sqid) {
295 g_free(sq);
296 }
297}
298
299static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
300{
301 NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
302 NvmeRequest *req, *next;
303 NvmeSQueue *sq;
304 NvmeCQueue *cq;
305 uint16_t qid = le16_to_cpu(c->qid);
306
307 if (!qid || nvme_check_sqid(n, qid)) {
308 return NVME_INVALID_QID | NVME_DNR;
309 }
310
311 sq = n->sq[qid];
312 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
313 req = QTAILQ_FIRST(&sq->out_req_list);
314 assert(req->aiocb);
315 blk_aio_cancel(req->aiocb);
316 }
317 if (!nvme_check_cqid(n, sq->cqid)) {
318 cq = n->cq[sq->cqid];
319 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
320
321 nvme_post_cqes(cq);
322 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
323 if (req->sq == sq) {
324 QTAILQ_REMOVE(&cq->req_list, req, entry);
325 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
326 }
327 }
328 }
329
330 nvme_free_sq(sq, n);
331 return NVME_SUCCESS;
332}
333
334static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
335 uint16_t sqid, uint16_t cqid, uint16_t size)
336{
337 int i;
338 NvmeCQueue *cq;
339
340 sq->ctrl = n;
341 sq->dma_addr = dma_addr;
342 sq->sqid = sqid;
343 sq->size = size;
344 sq->cqid = cqid;
345 sq->head = sq->tail = 0;
346 sq->io_req = g_new(NvmeRequest, sq->size);
347
348 QTAILQ_INIT(&sq->req_list);
349 QTAILQ_INIT(&sq->out_req_list);
350 for (i = 0; i < sq->size; i++) {
351 sq->io_req[i].sq = sq;
352 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
353 }
354 sq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_process_sq, sq);
355
356 assert(n->cq[cqid]);
357 cq = n->cq[cqid];
358 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
359 n->sq[sqid] = sq;
360}
361
362static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
363{
364 NvmeSQueue *sq;
365 NvmeCreateSq *c = (NvmeCreateSq *)cmd;
366
367 uint16_t cqid = le16_to_cpu(c->cqid);
368 uint16_t sqid = le16_to_cpu(c->sqid);
369 uint16_t qsize = le16_to_cpu(c->qsize);
370 uint16_t qflags = le16_to_cpu(c->sq_flags);
371 uint64_t prp1 = le64_to_cpu(c->prp1);
372
373 if (!cqid || nvme_check_cqid(n, cqid)) {
374 return NVME_INVALID_CQID | NVME_DNR;
375 }
376 if (!sqid || (sqid && !nvme_check_sqid(n, sqid))) {
377 return NVME_INVALID_QID | NVME_DNR;
378 }
379 if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
380 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
381 }
382 if (!prp1 || prp1 & (n->page_size - 1)) {
383 return NVME_INVALID_FIELD | NVME_DNR;
384 }
385 if (!(NVME_SQ_FLAGS_PC(qflags))) {
386 return NVME_INVALID_FIELD | NVME_DNR;
387 }
388 sq = g_malloc0(sizeof(*sq));
389 nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
390 return NVME_SUCCESS;
391}
392
393static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
394{
395 n->cq[cq->cqid] = NULL;
396 timer_del(cq->timer);
397 timer_free(cq->timer);
398 msix_vector_unuse(&n->parent_obj, cq->vector);
399 if (cq->cqid) {
400 g_free(cq);
401 }
402}
403
404static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
405{
406 NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
407 NvmeCQueue *cq;
408 uint16_t qid = le16_to_cpu(c->qid);
409
410 if (!qid || nvme_check_cqid(n, qid)) {
411 return NVME_INVALID_CQID | NVME_DNR;
412 }
413
414 cq = n->cq[qid];
415 if (!QTAILQ_EMPTY(&cq->sq_list)) {
416 return NVME_INVALID_QUEUE_DEL;
417 }
418 nvme_free_cq(cq, n);
419 return NVME_SUCCESS;
420}
421
422static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
423 uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled)
424{
425 cq->ctrl = n;
426 cq->cqid = cqid;
427 cq->size = size;
428 cq->dma_addr = dma_addr;
429 cq->phase = 1;
430 cq->irq_enabled = irq_enabled;
431 cq->vector = vector;
432 cq->head = cq->tail = 0;
433 QTAILQ_INIT(&cq->req_list);
434 QTAILQ_INIT(&cq->sq_list);
435 msix_vector_use(&n->parent_obj, cq->vector);
436 n->cq[cqid] = cq;
437 cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
438}
439
440static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
441{
442 NvmeCQueue *cq;
443 NvmeCreateCq *c = (NvmeCreateCq *)cmd;
444 uint16_t cqid = le16_to_cpu(c->cqid);
445 uint16_t vector = le16_to_cpu(c->irq_vector);
446 uint16_t qsize = le16_to_cpu(c->qsize);
447 uint16_t qflags = le16_to_cpu(c->cq_flags);
448 uint64_t prp1 = le64_to_cpu(c->prp1);
449
450 if (!cqid || (cqid && !nvme_check_cqid(n, cqid))) {
451 return NVME_INVALID_CQID | NVME_DNR;
452 }
453 if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
454 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
455 }
456 if (!prp1) {
457 return NVME_INVALID_FIELD | NVME_DNR;
458 }
459 if (vector > n->num_queues) {
460 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
461 }
462 if (!(NVME_CQ_FLAGS_PC(qflags))) {
463 return NVME_INVALID_FIELD | NVME_DNR;
464 }
465
466 cq = g_malloc0(sizeof(*cq));
467 nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
468 NVME_CQ_FLAGS_IEN(qflags));
469 return NVME_SUCCESS;
470}
471
472static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
473{
474 NvmeNamespace *ns;
475 NvmeIdentify *c = (NvmeIdentify *)cmd;
476 uint32_t cns = le32_to_cpu(c->cns);
477 uint32_t nsid = le32_to_cpu(c->nsid);
478 uint64_t prp1 = le64_to_cpu(c->prp1);
479 uint64_t prp2 = le64_to_cpu(c->prp2);
480
481 if (cns) {
482 return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
483 prp1, prp2);
484 }
485 if (nsid == 0 || nsid > n->num_namespaces) {
486 return NVME_INVALID_NSID | NVME_DNR;
487 }
488
489 ns = &n->namespaces[nsid - 1];
490 return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
491 prp1, prp2);
492}
493
494static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
495{
496 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
497 uint32_t result;
498
499 switch (dw10) {
500 case NVME_VOLATILE_WRITE_CACHE:
501 result = blk_enable_write_cache(n->conf.blk);
502 break;
503 case NVME_NUMBER_OF_QUEUES:
504 result = cpu_to_le32((n->num_queues - 1) | ((n->num_queues - 1) << 16));
505 break;
506 default:
507 return NVME_INVALID_FIELD | NVME_DNR;
508 }
509
510 req->cqe.result = result;
511 return NVME_SUCCESS;
512}
513
514static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
515{
516 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
517 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
518
519 switch (dw10) {
520 case NVME_VOLATILE_WRITE_CACHE:
521 blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
522 break;
523 case NVME_NUMBER_OF_QUEUES:
524 req->cqe.result =
525 cpu_to_le32((n->num_queues - 1) | ((n->num_queues - 1) << 16));
526 break;
527 default:
528 return NVME_INVALID_FIELD | NVME_DNR;
529 }
530 return NVME_SUCCESS;
531}
532
533static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
534{
535 switch (cmd->opcode) {
536 case NVME_ADM_CMD_DELETE_SQ:
537 return nvme_del_sq(n, cmd);
538 case NVME_ADM_CMD_CREATE_SQ:
539 return nvme_create_sq(n, cmd);
540 case NVME_ADM_CMD_DELETE_CQ:
541 return nvme_del_cq(n, cmd);
542 case NVME_ADM_CMD_CREATE_CQ:
543 return nvme_create_cq(n, cmd);
544 case NVME_ADM_CMD_IDENTIFY:
545 return nvme_identify(n, cmd);
546 case NVME_ADM_CMD_SET_FEATURES:
547 return nvme_set_feature(n, cmd, req);
548 case NVME_ADM_CMD_GET_FEATURES:
549 return nvme_get_feature(n, cmd, req);
550 default:
551 return NVME_INVALID_OPCODE | NVME_DNR;
552 }
553}
554
555static void nvme_process_sq(void *opaque)
556{
557 NvmeSQueue *sq = opaque;
558 NvmeCtrl *n = sq->ctrl;
559 NvmeCQueue *cq = n->cq[sq->cqid];
560
561 uint16_t status;
562 hwaddr addr;
563 NvmeCmd cmd;
564 NvmeRequest *req;
565
566 while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
567 addr = sq->dma_addr + sq->head * n->sqe_size;
568 pci_dma_read(&n->parent_obj, addr, (void *)&cmd, sizeof(cmd));
569 nvme_inc_sq_head(sq);
570
571 req = QTAILQ_FIRST(&sq->req_list);
572 QTAILQ_REMOVE(&sq->req_list, req, entry);
573 QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
574 memset(&req->cqe, 0, sizeof(req->cqe));
575 req->cqe.cid = cmd.cid;
576
577 status = sq->sqid ? nvme_io_cmd(n, &cmd, req) :
578 nvme_admin_cmd(n, &cmd, req);
579 if (status != NVME_NO_COMPLETE) {
580 req->status = status;
581 nvme_enqueue_req_completion(cq, req);
582 }
583 }
584}
585
586static void nvme_clear_ctrl(NvmeCtrl *n)
587{
588 int i;
589
590 for (i = 0; i < n->num_queues; i++) {
591 if (n->sq[i] != NULL) {
592 nvme_free_sq(n->sq[i], n);
593 }
594 }
595 for (i = 0; i < n->num_queues; i++) {
596 if (n->cq[i] != NULL) {
597 nvme_free_cq(n->cq[i], n);
598 }
599 }
600
601 blk_flush(n->conf.blk);
602 n->bar.cc = 0;
603}
604
605static int nvme_start_ctrl(NvmeCtrl *n)
606{
607 uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
608 uint32_t page_size = 1 << page_bits;
609
610 if (n->cq[0] || n->sq[0] || !n->bar.asq || !n->bar.acq ||
611 n->bar.asq & (page_size - 1) || n->bar.acq & (page_size - 1) ||
612 NVME_CC_MPS(n->bar.cc) < NVME_CAP_MPSMIN(n->bar.cap) ||
613 NVME_CC_MPS(n->bar.cc) > NVME_CAP_MPSMAX(n->bar.cap) ||
614 NVME_CC_IOCQES(n->bar.cc) < NVME_CTRL_CQES_MIN(n->id_ctrl.cqes) ||
615 NVME_CC_IOCQES(n->bar.cc) > NVME_CTRL_CQES_MAX(n->id_ctrl.cqes) ||
616 NVME_CC_IOSQES(n->bar.cc) < NVME_CTRL_SQES_MIN(n->id_ctrl.sqes) ||
617 NVME_CC_IOSQES(n->bar.cc) > NVME_CTRL_SQES_MAX(n->id_ctrl.sqes) ||
618 !NVME_AQA_ASQS(n->bar.aqa) || !NVME_AQA_ACQS(n->bar.aqa)) {
619 return -1;
620 }
621
622 n->page_bits = page_bits;
623 n->page_size = page_size;
624 n->max_prp_ents = n->page_size / sizeof(uint64_t);
625 n->cqe_size = 1 << NVME_CC_IOCQES(n->bar.cc);
626 n->sqe_size = 1 << NVME_CC_IOSQES(n->bar.cc);
627 nvme_init_cq(&n->admin_cq, n, n->bar.acq, 0, 0,
628 NVME_AQA_ACQS(n->bar.aqa) + 1, 1);
629 nvme_init_sq(&n->admin_sq, n, n->bar.asq, 0, 0,
630 NVME_AQA_ASQS(n->bar.aqa) + 1);
631
632 return 0;
633}
634
635static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
636 unsigned size)
637{
638 switch (offset) {
639 case 0xc:
640 n->bar.intms |= data & 0xffffffff;
641 n->bar.intmc = n->bar.intms;
642 break;
643 case 0x10:
644 n->bar.intms &= ~(data & 0xffffffff);
645 n->bar.intmc = n->bar.intms;
646 break;
647 case 0x14:
648
649 if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
650 !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
651 {
652 n->bar.cc = data;
653 }
654
655 if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
656 n->bar.cc = data;
657 if (nvme_start_ctrl(n)) {
658 n->bar.csts = NVME_CSTS_FAILED;
659 } else {
660 n->bar.csts = NVME_CSTS_READY;
661 }
662 } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
663 nvme_clear_ctrl(n);
664 n->bar.csts &= ~NVME_CSTS_READY;
665 }
666 if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
667 nvme_clear_ctrl(n);
668 n->bar.cc = data;
669 n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
670 } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
671 n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
672 n->bar.cc = data;
673 }
674 break;
675 case 0x24:
676 n->bar.aqa = data & 0xffffffff;
677 break;
678 case 0x28:
679 n->bar.asq = data;
680 break;
681 case 0x2c:
682 n->bar.asq |= data << 32;
683 break;
684 case 0x30:
685 n->bar.acq = data;
686 break;
687 case 0x34:
688 n->bar.acq |= data << 32;
689 break;
690 default:
691 break;
692 }
693}
694
695static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
696{
697 NvmeCtrl *n = (NvmeCtrl *)opaque;
698 uint8_t *ptr = (uint8_t *)&n->bar;
699 uint64_t val = 0;
700
701 if (addr < sizeof(n->bar)) {
702 memcpy(&val, ptr + addr, size);
703 }
704 return val;
705}
706
707static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
708{
709 uint32_t qid;
710
711 if (addr & ((1 << 2) - 1)) {
712 return;
713 }
714
715 if (((addr - 0x1000) >> 2) & 1) {
716 uint16_t new_head = val & 0xffff;
717 int start_sqs;
718 NvmeCQueue *cq;
719
720 qid = (addr - (0x1000 + (1 << 2))) >> 3;
721 if (nvme_check_cqid(n, qid)) {
722 return;
723 }
724
725 cq = n->cq[qid];
726 if (new_head >= cq->size) {
727 return;
728 }
729
730 start_sqs = nvme_cq_full(cq) ? 1 : 0;
731 cq->head = new_head;
732 if (start_sqs) {
733 NvmeSQueue *sq;
734 QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
735 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
736 }
737 timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
738 }
739
740 if (cq->tail != cq->head) {
741 nvme_isr_notify(n, cq);
742 }
743 } else {
744 uint16_t new_tail = val & 0xffff;
745 NvmeSQueue *sq;
746
747 qid = (addr - 0x1000) >> 3;
748 if (nvme_check_sqid(n, qid)) {
749 return;
750 }
751
752 sq = n->sq[qid];
753 if (new_tail >= sq->size) {
754 return;
755 }
756
757 sq->tail = new_tail;
758 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
759 }
760}
761
762static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
763 unsigned size)
764{
765 NvmeCtrl *n = (NvmeCtrl *)opaque;
766 if (addr < sizeof(n->bar)) {
767 nvme_write_bar(n, addr, data, size);
768 } else if (addr >= 0x1000) {
769 nvme_process_db(n, addr, data);
770 }
771}
772
773static const MemoryRegionOps nvme_mmio_ops = {
774 .read = nvme_mmio_read,
775 .write = nvme_mmio_write,
776 .endianness = DEVICE_LITTLE_ENDIAN,
777 .impl = {
778 .min_access_size = 2,
779 .max_access_size = 8,
780 },
781};
782
783static int nvme_init(PCIDevice *pci_dev)
784{
785 NvmeCtrl *n = NVME(pci_dev);
786 NvmeIdCtrl *id = &n->id_ctrl;
787
788 int i;
789 int64_t bs_size;
790 uint8_t *pci_conf;
791
792 if (!n->conf.blk) {
793 return -1;
794 }
795
796 bs_size = blk_getlength(n->conf.blk);
797 if (bs_size < 0) {
798 return -1;
799 }
800
801 blkconf_serial(&n->conf, &n->serial);
802 if (!n->serial) {
803 return -1;
804 }
805 blkconf_blocksizes(&n->conf);
806
807 pci_conf = pci_dev->config;
808 pci_conf[PCI_INTERRUPT_PIN] = 1;
809 pci_config_set_prog_interface(pci_dev->config, 0x2);
810 pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
811 pcie_endpoint_cap_init(&n->parent_obj, 0x80);
812
813 n->num_namespaces = 1;
814 n->num_queues = 64;
815 n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
816 n->ns_size = bs_size / (uint64_t)n->num_namespaces;
817
818 n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
819 n->sq = g_new0(NvmeSQueue *, n->num_queues);
820 n->cq = g_new0(NvmeCQueue *, n->num_queues);
821
822 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n,
823 "nvme", n->reg_size);
824 pci_register_bar(&n->parent_obj, 0,
825 PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64,
826 &n->iomem);
827 msix_init_exclusive_bar(&n->parent_obj, n->num_queues, 4);
828
829 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
830 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
831 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
832 strpadcpy((char *)id->fr, sizeof(id->fr), "1.0", ' ');
833 strpadcpy((char *)id->sn, sizeof(id->sn), n->serial, ' ');
834 id->rab = 6;
835 id->ieee[0] = 0x00;
836 id->ieee[1] = 0x02;
837 id->ieee[2] = 0xb3;
838 id->oacs = cpu_to_le16(0);
839 id->frmw = 7 << 1;
840 id->lpa = 1 << 0;
841 id->sqes = (0x6 << 4) | 0x6;
842 id->cqes = (0x4 << 4) | 0x4;
843 id->nn = cpu_to_le32(n->num_namespaces);
844 id->psd[0].mp = cpu_to_le16(0x9c4);
845 id->psd[0].enlat = cpu_to_le32(0x10);
846 id->psd[0].exlat = cpu_to_le32(0x4);
847 if (blk_enable_write_cache(n->conf.blk)) {
848 id->vwc = 1;
849 }
850
851 n->bar.cap = 0;
852 NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
853 NVME_CAP_SET_CQR(n->bar.cap, 1);
854 NVME_CAP_SET_AMS(n->bar.cap, 1);
855 NVME_CAP_SET_TO(n->bar.cap, 0xf);
856 NVME_CAP_SET_CSS(n->bar.cap, 1);
857 NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
858
859 n->bar.vs = 0x00010100;
860 n->bar.intmc = n->bar.intms = 0;
861
862 for (i = 0; i < n->num_namespaces; i++) {
863 NvmeNamespace *ns = &n->namespaces[i];
864 NvmeIdNs *id_ns = &ns->id_ns;
865 id_ns->nsfeat = 0;
866 id_ns->nlbaf = 0;
867 id_ns->flbas = 0;
868 id_ns->mc = 0;
869 id_ns->dpc = 0;
870 id_ns->dps = 0;
871 id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
872 id_ns->ncap = id_ns->nuse = id_ns->nsze =
873 cpu_to_le64(n->ns_size >>
874 id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
875 }
876 return 0;
877}
878
879static void nvme_exit(PCIDevice *pci_dev)
880{
881 NvmeCtrl *n = NVME(pci_dev);
882
883 nvme_clear_ctrl(n);
884 g_free(n->namespaces);
885 g_free(n->cq);
886 g_free(n->sq);
887 msix_uninit_exclusive_bar(pci_dev);
888}
889
890static Property nvme_props[] = {
891 DEFINE_BLOCK_PROPERTIES(NvmeCtrl, conf),
892 DEFINE_PROP_STRING("serial", NvmeCtrl, serial),
893 DEFINE_PROP_END_OF_LIST(),
894};
895
896static const VMStateDescription nvme_vmstate = {
897 .name = "nvme",
898 .unmigratable = 1,
899};
900
901static void nvme_class_init(ObjectClass *oc, void *data)
902{
903 DeviceClass *dc = DEVICE_CLASS(oc);
904 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
905
906 pc->init = nvme_init;
907 pc->exit = nvme_exit;
908 pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
909 pc->vendor_id = PCI_VENDOR_ID_INTEL;
910 pc->device_id = 0x5845;
911 pc->revision = 1;
912 pc->is_express = 1;
913
914 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
915 dc->desc = "Non-Volatile Memory Express";
916 dc->props = nvme_props;
917 dc->vmsd = &nvme_vmstate;
918}
919
920static void nvme_instance_init(Object *obj)
921{
922 NvmeCtrl *s = NVME(obj);
923
924 device_add_bootindex_property(obj, &s->conf.bootindex,
925 "bootindex", "/namespace@1,0",
926 DEVICE(obj), &error_abort);
927}
928
929static const TypeInfo nvme_info = {
930 .name = "nvme",
931 .parent = TYPE_PCI_DEVICE,
932 .instance_size = sizeof(NvmeCtrl),
933 .class_init = nvme_class_init,
934 .instance_init = nvme_instance_init,
935};
936
937static void nvme_register_types(void)
938{
939 type_register_static(&nvme_info);
940}
941
942type_init(nvme_register_types)
943
