1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include "qemu/osdep.h"
24#include "hw/block/block.h"
25#include "hw/hw.h"
26#include "hw/pci/msix.h"
27#include "hw/pci/pci.h"
28#include "sysemu/sysemu.h"
29#include "qapi/error.h"
30#include "qapi/visitor.h"
31#include "sysemu/block-backend.h"
32
33#include "nvme.h"
34
35static void nvme_process_sq(void *opaque);
36
37static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
38{
39 return sqid < n->num_queues && n->sq[sqid] != NULL ? 0 : -1;
40}
41
42static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
43{
44 return cqid < n->num_queues && n->cq[cqid] != NULL ? 0 : -1;
45}
46
47static void nvme_inc_cq_tail(NvmeCQueue *cq)
48{
49 cq->tail++;
50 if (cq->tail >= cq->size) {
51 cq->tail = 0;
52 cq->phase = !cq->phase;
53 }
54}
55
56static void nvme_inc_sq_head(NvmeSQueue *sq)
57{
58 sq->head = (sq->head + 1) % sq->size;
59}
60
61static uint8_t nvme_cq_full(NvmeCQueue *cq)
62{
63 return (cq->tail + 1) % cq->size == cq->head;
64}
65
66static uint8_t nvme_sq_empty(NvmeSQueue *sq)
67{
68 return sq->head == sq->tail;
69}
70
71static void nvme_isr_notify(NvmeCtrl *n, NvmeCQueue *cq)
72{
73 if (cq->irq_enabled) {
74 if (msix_enabled(&(n->parent_obj))) {
75 msix_notify(&(n->parent_obj), cq->vector);
76 } else {
77 pci_irq_pulse(&n->parent_obj);
78 }
79 }
80}
81
82static uint16_t nvme_map_prp(QEMUSGList *qsg, uint64_t prp1, uint64_t prp2,
83 uint32_t len, NvmeCtrl *n)
84{
85 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
86 trans_len = MIN(len, trans_len);
87 int num_prps = (len >> n->page_bits) + 1;
88
89 if (!prp1) {
90 return NVME_INVALID_FIELD | NVME_DNR;
91 }
92
93 pci_dma_sglist_init(qsg, &n->parent_obj, num_prps);
94 qemu_sglist_add(qsg, prp1, trans_len);
95 len -= trans_len;
96 if (len) {
97 if (!prp2) {
98 goto unmap;
99 }
100 if (len > n->page_size) {
101 uint64_t prp_list[n->max_prp_ents];
102 uint32_t nents, prp_trans;
103 int i = 0;
104
105 nents = (len + n->page_size - 1) >> n->page_bits;
106 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
107 pci_dma_read(&n->parent_obj, prp2, (void *)prp_list, prp_trans);
108 while (len != 0) {
109 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
110
111 if (i == n->max_prp_ents - 1 && len > n->page_size) {
112 if (!prp_ent || prp_ent & (n->page_size - 1)) {
113 goto unmap;
114 }
115
116 i = 0;
117 nents = (len + n->page_size - 1) >> n->page_bits;
118 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
119 pci_dma_read(&n->parent_obj, prp_ent, (void *)prp_list,
120 prp_trans);
121 prp_ent = le64_to_cpu(prp_list[i]);
122 }
123
124 if (!prp_ent || prp_ent & (n->page_size - 1)) {
125 goto unmap;
126 }
127
128 trans_len = MIN(len, n->page_size);
129 qemu_sglist_add(qsg, prp_ent, trans_len);
130 len -= trans_len;
131 i++;
132 }
133 } else {
134 if (prp2 & (n->page_size - 1)) {
135 goto unmap;
136 }
137 qemu_sglist_add(qsg, prp2, len);
138 }
139 }
140 return NVME_SUCCESS;
141
142 unmap:
143 qemu_sglist_destroy(qsg);
144 return NVME_INVALID_FIELD | NVME_DNR;
145}
146
147static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
148 uint64_t prp1, uint64_t prp2)
149{
150 QEMUSGList qsg;
151
152 if (nvme_map_prp(&qsg, prp1, prp2, len, n)) {
153 return NVME_INVALID_FIELD | NVME_DNR;
154 }
155 if (dma_buf_read(ptr, len, &qsg)) {
156 qemu_sglist_destroy(&qsg);
157 return NVME_INVALID_FIELD | NVME_DNR;
158 }
159 qemu_sglist_destroy(&qsg);
160 return NVME_SUCCESS;
161}
162
163static void nvme_post_cqes(void *opaque)
164{
165 NvmeCQueue *cq = opaque;
166 NvmeCtrl *n = cq->ctrl;
167 NvmeRequest *req, *next;
168
169 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
170 NvmeSQueue *sq;
171 hwaddr addr;
172
173 if (nvme_cq_full(cq)) {
174 break;
175 }
176
177 QTAILQ_REMOVE(&cq->req_list, req, entry);
178 sq = req->sq;
179 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
180 req->cqe.sq_id = cpu_to_le16(sq->sqid);
181 req->cqe.sq_head = cpu_to_le16(sq->head);
182 addr = cq->dma_addr + cq->tail * n->cqe_size;
183 nvme_inc_cq_tail(cq);
184 pci_dma_write(&n->parent_obj, addr, (void *)&req->cqe,
185 sizeof(req->cqe));
186 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
187 }
188 nvme_isr_notify(n, cq);
189}
190
191static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
192{
193 assert(cq->cqid == req->sq->cqid);
194 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
195 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
196 timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
197}
198
199static void nvme_rw_cb(void *opaque, int ret)
200{
201 NvmeRequest *req = opaque;
202 NvmeSQueue *sq = req->sq;
203 NvmeCtrl *n = sq->ctrl;
204 NvmeCQueue *cq = n->cq[sq->cqid];
205
206 if (!ret) {
207 block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
208 req->status = NVME_SUCCESS;
209 } else {
210 block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
211 req->status = NVME_INTERNAL_DEV_ERROR;
212 }
213 if (req->has_sg) {
214 qemu_sglist_destroy(&req->qsg);
215 }
216 nvme_enqueue_req_completion(cq, req);
217}
218
219static uint16_t nvme_flush(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
220 NvmeRequest *req)
221{
222 req->has_sg = false;
223 block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
224 BLOCK_ACCT_FLUSH);
225 req->aiocb = blk_aio_flush(n->conf.blk, nvme_rw_cb, req);
226
227 return NVME_NO_COMPLETE;
228}
229
230static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
231 NvmeRequest *req)
232{
233 NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
234 uint32_t nlb = le32_to_cpu(rw->nlb) + 1;
235 uint64_t slba = le64_to_cpu(rw->slba);
236 uint64_t prp1 = le64_to_cpu(rw->prp1);
237 uint64_t prp2 = le64_to_cpu(rw->prp2);
238
239 uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
240 uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
241 uint64_t data_size = (uint64_t)nlb << data_shift;
242 uint64_t data_offset = slba << data_shift;
243 int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
244 enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
245
246 if ((slba + nlb) > ns->id_ns.nsze) {
247 block_acct_invalid(blk_get_stats(n->conf.blk), acct);
248 return NVME_LBA_RANGE | NVME_DNR;
249 }
250
251 if (nvme_map_prp(&req->qsg, prp1, prp2, data_size, n)) {
252 block_acct_invalid(blk_get_stats(n->conf.blk), acct);
253 return NVME_INVALID_FIELD | NVME_DNR;
254 }
255
256 assert((nlb << data_shift) == req->qsg.size);
257
258 req->has_sg = true;
259 dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
260 req->aiocb = is_write ?
261 dma_blk_write(n->conf.blk, &req->qsg, data_offset, nvme_rw_cb, req) :
262 dma_blk_read(n->conf.blk, &req->qsg, data_offset, nvme_rw_cb, req);
263
264 return NVME_NO_COMPLETE;
265}
266
267static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
268{
269 NvmeNamespace *ns;
270 uint32_t nsid = le32_to_cpu(cmd->nsid);
271
272 if (nsid == 0 || nsid > n->num_namespaces) {
273 return NVME_INVALID_NSID | NVME_DNR;
274 }
275
276 ns = &n->namespaces[nsid - 1];
277 switch (cmd->opcode) {
278 case NVME_CMD_FLUSH:
279 return nvme_flush(n, ns, cmd, req);
280 case NVME_CMD_WRITE:
281 case NVME_CMD_READ:
282 return nvme_rw(n, ns, cmd, req);
283 default:
284 return NVME_INVALID_OPCODE | NVME_DNR;
285 }
286}
287
288static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
289{
290 n->sq[sq->sqid] = NULL;
291 timer_del(sq->timer);
292 timer_free(sq->timer);
293 g_free(sq->io_req);
294 if (sq->sqid) {
295 g_free(sq);
296 }
297}
298
299static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
300{
301 NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
302 NvmeRequest *req, *next;
303 NvmeSQueue *sq;
304 NvmeCQueue *cq;
305 uint16_t qid = le16_to_cpu(c->qid);
306
307 if (!qid || nvme_check_sqid(n, qid)) {
308 return NVME_INVALID_QID | NVME_DNR;
309 }
310
311 sq = n->sq[qid];
312 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
313 req = QTAILQ_FIRST(&sq->out_req_list);
314 assert(req->aiocb);
315 blk_aio_cancel(req->aiocb);
316 }
317 if (!nvme_check_cqid(n, sq->cqid)) {
318 cq = n->cq[sq->cqid];
319 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
320
321 nvme_post_cqes(cq);
322 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
323 if (req->sq == sq) {
324 QTAILQ_REMOVE(&cq->req_list, req, entry);
325 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
326 }
327 }
328 }
329
330 nvme_free_sq(sq, n);
331 return NVME_SUCCESS;
332}
333
334static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
335 uint16_t sqid, uint16_t cqid, uint16_t size)
336{
337 int i;
338 NvmeCQueue *cq;
339
340 sq->ctrl = n;
341 sq->dma_addr = dma_addr;
342 sq->sqid = sqid;
343 sq->size = size;
344 sq->cqid = cqid;
345 sq->head = sq->tail = 0;
346 sq->io_req = g_new(NvmeRequest, sq->size);
347
348 QTAILQ_INIT(&sq->req_list);
349 QTAILQ_INIT(&sq->out_req_list);
350 for (i = 0; i < sq->size; i++) {
351 sq->io_req[i].sq = sq;
352 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
353 }
354 sq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_process_sq, sq);
355
356 assert(n->cq[cqid]);
357 cq = n->cq[cqid];
358 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
359 n->sq[sqid] = sq;
360}
361
362static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
363{
364 NvmeSQueue *sq;
365 NvmeCreateSq *c = (NvmeCreateSq *)cmd;
366
367 uint16_t cqid = le16_to_cpu(c->cqid);
368 uint16_t sqid = le16_to_cpu(c->sqid);
369 uint16_t qsize = le16_to_cpu(c->qsize);
370 uint16_t qflags = le16_to_cpu(c->sq_flags);
371 uint64_t prp1 = le64_to_cpu(c->prp1);
372
373 if (!cqid || nvme_check_cqid(n, cqid)) {
374 return NVME_INVALID_CQID | NVME_DNR;
375 }
376 if (!sqid || (sqid && !nvme_check_sqid(n, sqid))) {
377 return NVME_INVALID_QID | NVME_DNR;
378 }
379 if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
380 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
381 }
382 if (!prp1 || prp1 & (n->page_size - 1)) {
383 return NVME_INVALID_FIELD | NVME_DNR;
384 }
385 if (!(NVME_SQ_FLAGS_PC(qflags))) {
386 return NVME_INVALID_FIELD | NVME_DNR;
387 }
388 sq = g_malloc0(sizeof(*sq));
389 nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
390 return NVME_SUCCESS;
391}
392
393static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
394{
395 n->cq[cq->cqid] = NULL;
396 timer_del(cq->timer);
397 timer_free(cq->timer);
398 msix_vector_unuse(&n->parent_obj, cq->vector);
399 if (cq->cqid) {
400 g_free(cq);
401 }
402}
403
404static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
405{
406 NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
407 NvmeCQueue *cq;
408 uint16_t qid = le16_to_cpu(c->qid);
409
410 if (!qid || nvme_check_cqid(n, qid)) {
411 return NVME_INVALID_CQID | NVME_DNR;
412 }
413
414 cq = n->cq[qid];
415 if (!QTAILQ_EMPTY(&cq->sq_list)) {
416 return NVME_INVALID_QUEUE_DEL;
417 }
418 nvme_free_cq(cq, n);
419 return NVME_SUCCESS;
420}
421
422static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
423 uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled)
424{
425 cq->ctrl = n;
426 cq->cqid = cqid;
427 cq->size = size;
428 cq->dma_addr = dma_addr;
429 cq->phase = 1;
430 cq->irq_enabled = irq_enabled;
431 cq->vector = vector;
432 cq->head = cq->tail = 0;
433 QTAILQ_INIT(&cq->req_list);
434 QTAILQ_INIT(&cq->sq_list);
435 msix_vector_use(&n->parent_obj, cq->vector);
436 n->cq[cqid] = cq;
437 cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
438}
439
440static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
441{
442 NvmeCQueue *cq;
443 NvmeCreateCq *c = (NvmeCreateCq *)cmd;
444 uint16_t cqid = le16_to_cpu(c->cqid);
445 uint16_t vector = le16_to_cpu(c->irq_vector);
446 uint16_t qsize = le16_to_cpu(c->qsize);
447 uint16_t qflags = le16_to_cpu(c->cq_flags);
448 uint64_t prp1 = le64_to_cpu(c->prp1);
449
450 if (!cqid || (cqid && !nvme_check_cqid(n, cqid))) {
451 return NVME_INVALID_CQID | NVME_DNR;
452 }
453 if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
454 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
455 }
456 if (!prp1) {
457 return NVME_INVALID_FIELD | NVME_DNR;
458 }
459 if (vector > n->num_queues) {
460 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
461 }
462 if (!(NVME_CQ_FLAGS_PC(qflags))) {
463 return NVME_INVALID_FIELD | NVME_DNR;
464 }
465
466 cq = g_malloc0(sizeof(*cq));
467 nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
468 NVME_CQ_FLAGS_IEN(qflags));
469 return NVME_SUCCESS;
470}
471
472static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
473{
474 uint64_t prp1 = le64_to_cpu(c->prp1);
475 uint64_t prp2 = le64_to_cpu(c->prp2);
476
477 return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
478 prp1, prp2);
479}
480
481static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
482{
483 NvmeNamespace *ns;
484 uint32_t nsid = le32_to_cpu(c->nsid);
485 uint64_t prp1 = le64_to_cpu(c->prp1);
486 uint64_t prp2 = le64_to_cpu(c->prp2);
487
488 if (nsid == 0 || nsid > n->num_namespaces) {
489 return NVME_INVALID_NSID | NVME_DNR;
490 }
491
492 ns = &n->namespaces[nsid - 1];
493 return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
494 prp1, prp2);
495}
496
497static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
498{
499 static const int data_len = 4096;
500 uint32_t min_nsid = le32_to_cpu(c->nsid);
501 uint64_t prp1 = le64_to_cpu(c->prp1);
502 uint64_t prp2 = le64_to_cpu(c->prp2);
503 uint32_t *list;
504 uint16_t ret;
505 int i, j = 0;
506
507 list = g_malloc0(data_len);
508 for (i = 0; i < n->num_namespaces; i++) {
509 if (i < min_nsid) {
510 continue;
511 }
512 list[j++] = cpu_to_le32(i + 1);
513 if (j == data_len / sizeof(uint32_t)) {
514 break;
515 }
516 }
517 ret = nvme_dma_read_prp(n, (uint8_t *)list, data_len, prp1, prp2);
518 g_free(list);
519 return ret;
520}
521
522
523static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
524{
525 NvmeIdentify *c = (NvmeIdentify *)cmd;
526
527 switch (le32_to_cpu(c->cns)) {
528 case 0x00:
529 return nvme_identify_ns(n, c);
530 case 0x01:
531 return nvme_identify_ctrl(n, c);
532 case 0x02:
533 return nvme_identify_nslist(n, c);
534 default:
535 return NVME_INVALID_FIELD | NVME_DNR;
536 }
537}
538
539static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
540{
541 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
542 uint32_t result;
543
544 switch (dw10) {
545 case NVME_VOLATILE_WRITE_CACHE:
546 result = blk_enable_write_cache(n->conf.blk);
547 break;
548 case NVME_NUMBER_OF_QUEUES:
549 result = cpu_to_le32((n->num_queues - 1) | ((n->num_queues - 1) << 16));
550 break;
551 default:
552 return NVME_INVALID_FIELD | NVME_DNR;
553 }
554
555 req->cqe.result = result;
556 return NVME_SUCCESS;
557}
558
559static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
560{
561 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
562 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
563
564 switch (dw10) {
565 case NVME_VOLATILE_WRITE_CACHE:
566 blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
567 break;
568 case NVME_NUMBER_OF_QUEUES:
569 req->cqe.result =
570 cpu_to_le32((n->num_queues - 1) | ((n->num_queues - 1) << 16));
571 break;
572 default:
573 return NVME_INVALID_FIELD | NVME_DNR;
574 }
575 return NVME_SUCCESS;
576}
577
578static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
579{
580 switch (cmd->opcode) {
581 case NVME_ADM_CMD_DELETE_SQ:
582 return nvme_del_sq(n, cmd);
583 case NVME_ADM_CMD_CREATE_SQ:
584 return nvme_create_sq(n, cmd);
585 case NVME_ADM_CMD_DELETE_CQ:
586 return nvme_del_cq(n, cmd);
587 case NVME_ADM_CMD_CREATE_CQ:
588 return nvme_create_cq(n, cmd);
589 case NVME_ADM_CMD_IDENTIFY:
590 return nvme_identify(n, cmd);
591 case NVME_ADM_CMD_SET_FEATURES:
592 return nvme_set_feature(n, cmd, req);
593 case NVME_ADM_CMD_GET_FEATURES:
594 return nvme_get_feature(n, cmd, req);
595 default:
596 return NVME_INVALID_OPCODE | NVME_DNR;
597 }
598}
599
600static void nvme_process_sq(void *opaque)
601{
602 NvmeSQueue *sq = opaque;
603 NvmeCtrl *n = sq->ctrl;
604 NvmeCQueue *cq = n->cq[sq->cqid];
605
606 uint16_t status;
607 hwaddr addr;
608 NvmeCmd cmd;
609 NvmeRequest *req;
610
611 while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
612 addr = sq->dma_addr + sq->head * n->sqe_size;
613 pci_dma_read(&n->parent_obj, addr, (void *)&cmd, sizeof(cmd));
614 nvme_inc_sq_head(sq);
615
616 req = QTAILQ_FIRST(&sq->req_list);
617 QTAILQ_REMOVE(&sq->req_list, req, entry);
618 QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
619 memset(&req->cqe, 0, sizeof(req->cqe));
620 req->cqe.cid = cmd.cid;
621
622 status = sq->sqid ? nvme_io_cmd(n, &cmd, req) :
623 nvme_admin_cmd(n, &cmd, req);
624 if (status != NVME_NO_COMPLETE) {
625 req->status = status;
626 nvme_enqueue_req_completion(cq, req);
627 }
628 }
629}
630
631static void nvme_clear_ctrl(NvmeCtrl *n)
632{
633 int i;
634
635 for (i = 0; i < n->num_queues; i++) {
636 if (n->sq[i] != NULL) {
637 nvme_free_sq(n->sq[i], n);
638 }
639 }
640 for (i = 0; i < n->num_queues; i++) {
641 if (n->cq[i] != NULL) {
642 nvme_free_cq(n->cq[i], n);
643 }
644 }
645
646 blk_flush(n->conf.blk);
647 n->bar.cc = 0;
648}
649
650static int nvme_start_ctrl(NvmeCtrl *n)
651{
652 uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
653 uint32_t page_size = 1 << page_bits;
654
655 if (n->cq[0] || n->sq[0] || !n->bar.asq || !n->bar.acq ||
656 n->bar.asq & (page_size - 1) || n->bar.acq & (page_size - 1) ||
657 NVME_CC_MPS(n->bar.cc) < NVME_CAP_MPSMIN(n->bar.cap) ||
658 NVME_CC_MPS(n->bar.cc) > NVME_CAP_MPSMAX(n->bar.cap) ||
659 NVME_CC_IOCQES(n->bar.cc) < NVME_CTRL_CQES_MIN(n->id_ctrl.cqes) ||
660 NVME_CC_IOCQES(n->bar.cc) > NVME_CTRL_CQES_MAX(n->id_ctrl.cqes) ||
661 NVME_CC_IOSQES(n->bar.cc) < NVME_CTRL_SQES_MIN(n->id_ctrl.sqes) ||
662 NVME_CC_IOSQES(n->bar.cc) > NVME_CTRL_SQES_MAX(n->id_ctrl.sqes) ||
663 !NVME_AQA_ASQS(n->bar.aqa) || !NVME_AQA_ACQS(n->bar.aqa)) {
664 return -1;
665 }
666
667 n->page_bits = page_bits;
668 n->page_size = page_size;
669 n->max_prp_ents = n->page_size / sizeof(uint64_t);
670 n->cqe_size = 1 << NVME_CC_IOCQES(n->bar.cc);
671 n->sqe_size = 1 << NVME_CC_IOSQES(n->bar.cc);
672 nvme_init_cq(&n->admin_cq, n, n->bar.acq, 0, 0,
673 NVME_AQA_ACQS(n->bar.aqa) + 1, 1);
674 nvme_init_sq(&n->admin_sq, n, n->bar.asq, 0, 0,
675 NVME_AQA_ASQS(n->bar.aqa) + 1);
676
677 return 0;
678}
679
680static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
681 unsigned size)
682{
683 switch (offset) {
684 case 0xc:
685 n->bar.intms |= data & 0xffffffff;
686 n->bar.intmc = n->bar.intms;
687 break;
688 case 0x10:
689 n->bar.intms &= ~(data & 0xffffffff);
690 n->bar.intmc = n->bar.intms;
691 break;
692 case 0x14:
693
694 if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
695 !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
696 {
697 n->bar.cc = data;
698 }
699
700 if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
701 n->bar.cc = data;
702 if (nvme_start_ctrl(n)) {
703 n->bar.csts = NVME_CSTS_FAILED;
704 } else {
705 n->bar.csts = NVME_CSTS_READY;
706 }
707 } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
708 nvme_clear_ctrl(n);
709 n->bar.csts &= ~NVME_CSTS_READY;
710 }
711 if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
712 nvme_clear_ctrl(n);
713 n->bar.cc = data;
714 n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
715 } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
716 n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
717 n->bar.cc = data;
718 }
719 break;
720 case 0x24:
721 n->bar.aqa = data & 0xffffffff;
722 break;
723 case 0x28:
724 n->bar.asq = data;
725 break;
726 case 0x2c:
727 n->bar.asq |= data << 32;
728 break;
729 case 0x30:
730 n->bar.acq = data;
731 break;
732 case 0x34:
733 n->bar.acq |= data << 32;
734 break;
735 default:
736 break;
737 }
738}
739
740static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
741{
742 NvmeCtrl *n = (NvmeCtrl *)opaque;
743 uint8_t *ptr = (uint8_t *)&n->bar;
744 uint64_t val = 0;
745
746 if (addr < sizeof(n->bar)) {
747 memcpy(&val, ptr + addr, size);
748 }
749 return val;
750}
751
752static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
753{
754 uint32_t qid;
755
756 if (addr & ((1 << 2) - 1)) {
757 return;
758 }
759
760 if (((addr - 0x1000) >> 2) & 1) {
761 uint16_t new_head = val & 0xffff;
762 int start_sqs;
763 NvmeCQueue *cq;
764
765 qid = (addr - (0x1000 + (1 << 2))) >> 3;
766 if (nvme_check_cqid(n, qid)) {
767 return;
768 }
769
770 cq = n->cq[qid];
771 if (new_head >= cq->size) {
772 return;
773 }
774
775 start_sqs = nvme_cq_full(cq) ? 1 : 0;
776 cq->head = new_head;
777 if (start_sqs) {
778 NvmeSQueue *sq;
779 QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
780 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
781 }
782 timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
783 }
784
785 if (cq->tail != cq->head) {
786 nvme_isr_notify(n, cq);
787 }
788 } else {
789 uint16_t new_tail = val & 0xffff;
790 NvmeSQueue *sq;
791
792 qid = (addr - 0x1000) >> 3;
793 if (nvme_check_sqid(n, qid)) {
794 return;
795 }
796
797 sq = n->sq[qid];
798 if (new_tail >= sq->size) {
799 return;
800 }
801
802 sq->tail = new_tail;
803 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
804 }
805}
806
807static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
808 unsigned size)
809{
810 NvmeCtrl *n = (NvmeCtrl *)opaque;
811 if (addr < sizeof(n->bar)) {
812 nvme_write_bar(n, addr, data, size);
813 } else if (addr >= 0x1000) {
814 nvme_process_db(n, addr, data);
815 }
816}
817
818static const MemoryRegionOps nvme_mmio_ops = {
819 .read = nvme_mmio_read,
820 .write = nvme_mmio_write,
821 .endianness = DEVICE_LITTLE_ENDIAN,
822 .impl = {
823 .min_access_size = 2,
824 .max_access_size = 8,
825 },
826};
827
828static int nvme_init(PCIDevice *pci_dev)
829{
830 NvmeCtrl *n = NVME(pci_dev);
831 NvmeIdCtrl *id = &n->id_ctrl;
832
833 int i;
834 int64_t bs_size;
835 uint8_t *pci_conf;
836
837 if (!n->conf.blk) {
838 return -1;
839 }
840
841 bs_size = blk_getlength(n->conf.blk);
842 if (bs_size < 0) {
843 return -1;
844 }
845
846 blkconf_serial(&n->conf, &n->serial);
847 if (!n->serial) {
848 return -1;
849 }
850 blkconf_blocksizes(&n->conf);
851 blkconf_apply_backend_options(&n->conf);
852
853 pci_conf = pci_dev->config;
854 pci_conf[PCI_INTERRUPT_PIN] = 1;
855 pci_config_set_prog_interface(pci_dev->config, 0x2);
856 pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
857 pcie_endpoint_cap_init(&n->parent_obj, 0x80);
858
859 n->num_namespaces = 1;
860 n->num_queues = 64;
861 n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
862 n->ns_size = bs_size / (uint64_t)n->num_namespaces;
863
864 n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
865 n->sq = g_new0(NvmeSQueue *, n->num_queues);
866 n->cq = g_new0(NvmeCQueue *, n->num_queues);
867
868 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n,
869 "nvme", n->reg_size);
870 pci_register_bar(&n->parent_obj, 0,
871 PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64,
872 &n->iomem);
873 msix_init_exclusive_bar(&n->parent_obj, n->num_queues, 4);
874
875 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
876 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
877 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
878 strpadcpy((char *)id->fr, sizeof(id->fr), "1.0", ' ');
879 strpadcpy((char *)id->sn, sizeof(id->sn), n->serial, ' ');
880 id->rab = 6;
881 id->ieee[0] = 0x00;
882 id->ieee[1] = 0x02;
883 id->ieee[2] = 0xb3;
884 id->oacs = cpu_to_le16(0);
885 id->frmw = 7 << 1;
886 id->lpa = 1 << 0;
887 id->sqes = (0x6 << 4) | 0x6;
888 id->cqes = (0x4 << 4) | 0x4;
889 id->nn = cpu_to_le32(n->num_namespaces);
890 id->psd[0].mp = cpu_to_le16(0x9c4);
891 id->psd[0].enlat = cpu_to_le32(0x10);
892 id->psd[0].exlat = cpu_to_le32(0x4);
893 if (blk_enable_write_cache(n->conf.blk)) {
894 id->vwc = 1;
895 }
896
897 n->bar.cap = 0;
898 NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
899 NVME_CAP_SET_CQR(n->bar.cap, 1);
900 NVME_CAP_SET_AMS(n->bar.cap, 1);
901 NVME_CAP_SET_TO(n->bar.cap, 0xf);
902 NVME_CAP_SET_CSS(n->bar.cap, 1);
903 NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
904
905 n->bar.vs = 0x00010100;
906 n->bar.intmc = n->bar.intms = 0;
907
908 for (i = 0; i < n->num_namespaces; i++) {
909 NvmeNamespace *ns = &n->namespaces[i];
910 NvmeIdNs *id_ns = &ns->id_ns;
911 id_ns->nsfeat = 0;
912 id_ns->nlbaf = 0;
913 id_ns->flbas = 0;
914 id_ns->mc = 0;
915 id_ns->dpc = 0;
916 id_ns->dps = 0;
917 id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
918 id_ns->ncap = id_ns->nuse = id_ns->nsze =
919 cpu_to_le64(n->ns_size >>
920 id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
921 }
922 return 0;
923}
924
925static void nvme_exit(PCIDevice *pci_dev)
926{
927 NvmeCtrl *n = NVME(pci_dev);
928
929 nvme_clear_ctrl(n);
930 g_free(n->namespaces);
931 g_free(n->cq);
932 g_free(n->sq);
933 msix_uninit_exclusive_bar(pci_dev);
934}
935
936static Property nvme_props[] = {
937 DEFINE_BLOCK_PROPERTIES(NvmeCtrl, conf),
938 DEFINE_PROP_STRING("serial", NvmeCtrl, serial),
939 DEFINE_PROP_END_OF_LIST(),
940};
941
942static const VMStateDescription nvme_vmstate = {
943 .name = "nvme",
944 .unmigratable = 1,
945};
946
947static void nvme_class_init(ObjectClass *oc, void *data)
948{
949 DeviceClass *dc = DEVICE_CLASS(oc);
950 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
951
952 pc->init = nvme_init;
953 pc->exit = nvme_exit;
954 pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
955 pc->vendor_id = PCI_VENDOR_ID_INTEL;
956 pc->device_id = 0x5845;
957 pc->revision = 2;
958 pc->is_express = 1;
959
960 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
961 dc->desc = "Non-Volatile Memory Express";
962 dc->props = nvme_props;
963 dc->vmsd = &nvme_vmstate;
964}
965
966static void nvme_instance_init(Object *obj)
967{
968 NvmeCtrl *s = NVME(obj);
969
970 device_add_bootindex_property(obj, &s->conf.bootindex,
971 "bootindex", "/namespace@1,0",
972 DEVICE(obj), &error_abort);
973}
974
975static const TypeInfo nvme_info = {
976 .name = "nvme",
977 .parent = TYPE_PCI_DEVICE,
978 .instance_size = sizeof(NvmeCtrl),
979 .class_init = nvme_class_init,
980 .instance_init = nvme_instance_init,
981};
982
983static void nvme_register_types(void)
984{
985 type_register_static(&nvme_info);
986}
987
988type_init(nvme_register_types)
989
