1
2
3
4
5
6
7
8
9#include "nvme.h"
10
11#include <linux/nvme.h>
12#include <linux/bitops.h>
13#include <linux/lightnvm.h>
14#include <linux/vmalloc.h>
15#include <linux/sched/sysctl.h>
16#include <uapi/linux/lightnvm.h>
17
18enum nvme_nvm_admin_opcode {
19 nvme_nvm_admin_identity = 0xe2,
20 nvme_nvm_admin_get_bb_tbl = 0xf2,
21 nvme_nvm_admin_set_bb_tbl = 0xf1,
22};
23
24enum nvme_nvm_log_page {
25 NVME_NVM_LOG_REPORT_CHUNK = 0xca,
26};
27
28struct nvme_nvm_ph_rw {
29 __u8 opcode;
30 __u8 flags;
31 __u16 command_id;
32 __le32 nsid;
33 __u64 rsvd2;
34 __le64 metadata;
35 __le64 prp1;
36 __le64 prp2;
37 __le64 spba;
38 __le16 length;
39 __le16 control;
40 __le32 dsmgmt;
41 __le64 resv;
42};
43
44struct nvme_nvm_erase_blk {
45 __u8 opcode;
46 __u8 flags;
47 __u16 command_id;
48 __le32 nsid;
49 __u64 rsvd[2];
50 __le64 prp1;
51 __le64 prp2;
52 __le64 spba;
53 __le16 length;
54 __le16 control;
55 __le32 dsmgmt;
56 __le64 resv;
57};
58
59struct nvme_nvm_identity {
60 __u8 opcode;
61 __u8 flags;
62 __u16 command_id;
63 __le32 nsid;
64 __u64 rsvd[2];
65 __le64 prp1;
66 __le64 prp2;
67 __u32 rsvd11[6];
68};
69
70struct nvme_nvm_getbbtbl {
71 __u8 opcode;
72 __u8 flags;
73 __u16 command_id;
74 __le32 nsid;
75 __u64 rsvd[2];
76 __le64 prp1;
77 __le64 prp2;
78 __le64 spba;
79 __u32 rsvd4[4];
80};
81
82struct nvme_nvm_setbbtbl {
83 __u8 opcode;
84 __u8 flags;
85 __u16 command_id;
86 __le32 nsid;
87 __le64 rsvd[2];
88 __le64 prp1;
89 __le64 prp2;
90 __le64 spba;
91 __le16 nlb;
92 __u8 value;
93 __u8 rsvd3;
94 __u32 rsvd4[3];
95};
96
97struct nvme_nvm_command {
98 union {
99 struct nvme_common_command common;
100 struct nvme_nvm_ph_rw ph_rw;
101 struct nvme_nvm_erase_blk erase;
102 struct nvme_nvm_identity identity;
103 struct nvme_nvm_getbbtbl get_bb;
104 struct nvme_nvm_setbbtbl set_bb;
105 };
106};
107
108struct nvme_nvm_id12_grp {
109 __u8 mtype;
110 __u8 fmtype;
111 __le16 res16;
112 __u8 num_ch;
113 __u8 num_lun;
114 __u8 num_pln;
115 __u8 rsvd1;
116 __le16 num_chk;
117 __le16 num_pg;
118 __le16 fpg_sz;
119 __le16 csecs;
120 __le16 sos;
121 __le16 rsvd2;
122 __le32 trdt;
123 __le32 trdm;
124 __le32 tprt;
125 __le32 tprm;
126 __le32 tbet;
127 __le32 tbem;
128 __le32 mpos;
129 __le32 mccap;
130 __le16 cpar;
131 __u8 reserved[906];
132} __packed;
133
134struct nvme_nvm_id12_addrf {
135 __u8 ch_offset;
136 __u8 ch_len;
137 __u8 lun_offset;
138 __u8 lun_len;
139 __u8 pln_offset;
140 __u8 pln_len;
141 __u8 blk_offset;
142 __u8 blk_len;
143 __u8 pg_offset;
144 __u8 pg_len;
145 __u8 sec_offset;
146 __u8 sec_len;
147 __u8 res[4];
148} __packed;
149
150struct nvme_nvm_id12 {
151 __u8 ver_id;
152 __u8 vmnt;
153 __u8 cgrps;
154 __u8 res;
155 __le32 cap;
156 __le32 dom;
157 struct nvme_nvm_id12_addrf ppaf;
158 __u8 resv[228];
159 struct nvme_nvm_id12_grp grp;
160 __u8 resv2[2880];
161} __packed;
162
163struct nvme_nvm_bb_tbl {
164 __u8 tblid[4];
165 __le16 verid;
166 __le16 revid;
167 __le32 rvsd1;
168 __le32 tblks;
169 __le32 tfact;
170 __le32 tgrown;
171 __le32 tdresv;
172 __le32 thresv;
173 __le32 rsvd2[8];
174 __u8 blk[0];
175};
176
177struct nvme_nvm_id20_addrf {
178 __u8 grp_len;
179 __u8 pu_len;
180 __u8 chk_len;
181 __u8 lba_len;
182 __u8 resv[4];
183};
184
185struct nvme_nvm_id20 {
186 __u8 mjr;
187 __u8 mnr;
188 __u8 resv[6];
189
190 struct nvme_nvm_id20_addrf lbaf;
191
192 __le32 mccap;
193 __u8 resv2[12];
194
195 __u8 wit;
196 __u8 resv3[31];
197
198
199 __le16 num_grp;
200 __le16 num_pu;
201 __le32 num_chk;
202 __le32 clba;
203 __u8 resv4[52];
204
205
206 __le32 ws_min;
207 __le32 ws_opt;
208 __le32 mw_cunits;
209 __le32 maxoc;
210 __le32 maxocpu;
211 __u8 resv5[44];
212
213
214 __le32 trdt;
215 __le32 trdm;
216 __le32 twrt;
217 __le32 twrm;
218 __le32 tcrst;
219 __le32 tcrsm;
220 __u8 resv6[40];
221
222
223 __u8 resv7[2816];
224
225
226 __u8 vs[1024];
227};
228
229struct nvme_nvm_chk_meta {
230 __u8 state;
231 __u8 type;
232 __u8 wi;
233 __u8 rsvd[5];
234 __le64 slba;
235 __le64 cnlb;
236 __le64 wp;
237};
238
239
240
241
242static inline void _nvme_nvm_check_size(void)
243{
244 BUILD_BUG_ON(sizeof(struct nvme_nvm_identity) != 64);
245 BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw) != 64);
246 BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk) != 64);
247 BUILD_BUG_ON(sizeof(struct nvme_nvm_getbbtbl) != 64);
248 BUILD_BUG_ON(sizeof(struct nvme_nvm_setbbtbl) != 64);
249 BUILD_BUG_ON(sizeof(struct nvme_nvm_id12_grp) != 960);
250 BUILD_BUG_ON(sizeof(struct nvme_nvm_id12_addrf) != 16);
251 BUILD_BUG_ON(sizeof(struct nvme_nvm_id12) != NVME_IDENTIFY_DATA_SIZE);
252 BUILD_BUG_ON(sizeof(struct nvme_nvm_bb_tbl) != 64);
253 BUILD_BUG_ON(sizeof(struct nvme_nvm_id20_addrf) != 8);
254 BUILD_BUG_ON(sizeof(struct nvme_nvm_id20) != NVME_IDENTIFY_DATA_SIZE);
255 BUILD_BUG_ON(sizeof(struct nvme_nvm_chk_meta) != 32);
256 BUILD_BUG_ON(sizeof(struct nvme_nvm_chk_meta) !=
257 sizeof(struct nvm_chk_meta));
258}
259
260static void nvme_nvm_set_addr_12(struct nvm_addrf_12 *dst,
261 struct nvme_nvm_id12_addrf *src)
262{
263 dst->ch_len = src->ch_len;
264 dst->lun_len = src->lun_len;
265 dst->blk_len = src->blk_len;
266 dst->pg_len = src->pg_len;
267 dst->pln_len = src->pln_len;
268 dst->sec_len = src->sec_len;
269
270 dst->ch_offset = src->ch_offset;
271 dst->lun_offset = src->lun_offset;
272 dst->blk_offset = src->blk_offset;
273 dst->pg_offset = src->pg_offset;
274 dst->pln_offset = src->pln_offset;
275 dst->sec_offset = src->sec_offset;
276
277 dst->ch_mask = ((1ULL << dst->ch_len) - 1) << dst->ch_offset;
278 dst->lun_mask = ((1ULL << dst->lun_len) - 1) << dst->lun_offset;
279 dst->blk_mask = ((1ULL << dst->blk_len) - 1) << dst->blk_offset;
280 dst->pg_mask = ((1ULL << dst->pg_len) - 1) << dst->pg_offset;
281 dst->pln_mask = ((1ULL << dst->pln_len) - 1) << dst->pln_offset;
282 dst->sec_mask = ((1ULL << dst->sec_len) - 1) << dst->sec_offset;
283}
284
285static int nvme_nvm_setup_12(struct nvme_nvm_id12 *id,
286 struct nvm_geo *geo)
287{
288 struct nvme_nvm_id12_grp *src;
289 int sec_per_pg, sec_per_pl, pg_per_blk;
290
291 if (id->cgrps != 1)
292 return -EINVAL;
293
294 src = &id->grp;
295
296 if (src->mtype != 0) {
297 pr_err("nvm: memory type not supported\n");
298 return -EINVAL;
299 }
300
301
302 geo->major_ver_id = id->ver_id;
303 geo->minor_ver_id = 2;
304
305
306 geo->version = NVM_OCSSD_SPEC_12;
307
308 geo->num_ch = src->num_ch;
309 geo->num_lun = src->num_lun;
310 geo->all_luns = geo->num_ch * geo->num_lun;
311
312 geo->num_chk = le16_to_cpu(src->num_chk);
313
314 geo->csecs = le16_to_cpu(src->csecs);
315 geo->sos = le16_to_cpu(src->sos);
316
317 pg_per_blk = le16_to_cpu(src->num_pg);
318 sec_per_pg = le16_to_cpu(src->fpg_sz) / geo->csecs;
319 sec_per_pl = sec_per_pg * src->num_pln;
320 geo->clba = sec_per_pl * pg_per_blk;
321
322 geo->all_chunks = geo->all_luns * geo->num_chk;
323 geo->total_secs = geo->clba * geo->all_chunks;
324
325 geo->ws_min = sec_per_pg;
326 geo->ws_opt = sec_per_pg;
327 geo->mw_cunits = geo->ws_opt << 3;
328
329
330
331
332
333 geo->maxoc = geo->all_luns * geo->num_chk;
334 geo->maxocpu = geo->num_chk;
335
336 geo->mccap = le32_to_cpu(src->mccap);
337
338 geo->trdt = le32_to_cpu(src->trdt);
339 geo->trdm = le32_to_cpu(src->trdm);
340 geo->tprt = le32_to_cpu(src->tprt);
341 geo->tprm = le32_to_cpu(src->tprm);
342 geo->tbet = le32_to_cpu(src->tbet);
343 geo->tbem = le32_to_cpu(src->tbem);
344
345
346 geo->vmnt = id->vmnt;
347 geo->cap = le32_to_cpu(id->cap);
348 geo->dom = le32_to_cpu(id->dom);
349
350 geo->mtype = src->mtype;
351 geo->fmtype = src->fmtype;
352
353 geo->cpar = le16_to_cpu(src->cpar);
354 geo->mpos = le32_to_cpu(src->mpos);
355
356 geo->pln_mode = NVM_PLANE_SINGLE;
357
358 if (geo->mpos & 0x020202) {
359 geo->pln_mode = NVM_PLANE_DOUBLE;
360 geo->ws_opt <<= 1;
361 } else if (geo->mpos & 0x040404) {
362 geo->pln_mode = NVM_PLANE_QUAD;
363 geo->ws_opt <<= 2;
364 }
365
366 geo->num_pln = src->num_pln;
367 geo->num_pg = le16_to_cpu(src->num_pg);
368 geo->fpg_sz = le16_to_cpu(src->fpg_sz);
369
370 nvme_nvm_set_addr_12((struct nvm_addrf_12 *)&geo->addrf, &id->ppaf);
371
372 return 0;
373}
374
375static void nvme_nvm_set_addr_20(struct nvm_addrf *dst,
376 struct nvme_nvm_id20_addrf *src)
377{
378 dst->ch_len = src->grp_len;
379 dst->lun_len = src->pu_len;
380 dst->chk_len = src->chk_len;
381 dst->sec_len = src->lba_len;
382
383 dst->sec_offset = 0;
384 dst->chk_offset = dst->sec_len;
385 dst->lun_offset = dst->chk_offset + dst->chk_len;
386 dst->ch_offset = dst->lun_offset + dst->lun_len;
387
388 dst->ch_mask = ((1ULL << dst->ch_len) - 1) << dst->ch_offset;
389 dst->lun_mask = ((1ULL << dst->lun_len) - 1) << dst->lun_offset;
390 dst->chk_mask = ((1ULL << dst->chk_len) - 1) << dst->chk_offset;
391 dst->sec_mask = ((1ULL << dst->sec_len) - 1) << dst->sec_offset;
392}
393
394static int nvme_nvm_setup_20(struct nvme_nvm_id20 *id,
395 struct nvm_geo *geo)
396{
397 geo->major_ver_id = id->mjr;
398 geo->minor_ver_id = id->mnr;
399
400
401 geo->version = NVM_OCSSD_SPEC_20;
402
403 geo->num_ch = le16_to_cpu(id->num_grp);
404 geo->num_lun = le16_to_cpu(id->num_pu);
405 geo->all_luns = geo->num_ch * geo->num_lun;
406
407 geo->num_chk = le32_to_cpu(id->num_chk);
408 geo->clba = le32_to_cpu(id->clba);
409
410 geo->all_chunks = geo->all_luns * geo->num_chk;
411 geo->total_secs = geo->clba * geo->all_chunks;
412
413 geo->ws_min = le32_to_cpu(id->ws_min);
414 geo->ws_opt = le32_to_cpu(id->ws_opt);
415 geo->mw_cunits = le32_to_cpu(id->mw_cunits);
416 geo->maxoc = le32_to_cpu(id->maxoc);
417 geo->maxocpu = le32_to_cpu(id->maxocpu);
418
419 geo->trdt = le32_to_cpu(id->trdt);
420 geo->trdm = le32_to_cpu(id->trdm);
421 geo->tprt = le32_to_cpu(id->twrt);
422 geo->tprm = le32_to_cpu(id->twrm);
423 geo->tbet = le32_to_cpu(id->tcrst);
424 geo->tbem = le32_to_cpu(id->tcrsm);
425
426 nvme_nvm_set_addr_20(&geo->addrf, &id->lbaf);
427
428 return 0;
429}
430
431static int nvme_nvm_identity(struct nvm_dev *nvmdev)
432{
433 struct nvme_ns *ns = nvmdev->q->queuedata;
434 struct nvme_nvm_id12 *id;
435 struct nvme_nvm_command c = {};
436 int ret;
437
438 c.identity.opcode = nvme_nvm_admin_identity;
439 c.identity.nsid = cpu_to_le32(ns->head->ns_id);
440
441 id = kmalloc(sizeof(struct nvme_nvm_id12), GFP_KERNEL);
442 if (!id)
443 return -ENOMEM;
444
445 ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, (struct nvme_command *)&c,
446 id, sizeof(struct nvme_nvm_id12));
447 if (ret) {
448 ret = -EIO;
449 goto out;
450 }
451
452
453
454
455
456 switch (id->ver_id) {
457 case 1:
458 ret = nvme_nvm_setup_12(id, &nvmdev->geo);
459 break;
460 case 2:
461 ret = nvme_nvm_setup_20((struct nvme_nvm_id20 *)id,
462 &nvmdev->geo);
463 break;
464 default:
465 dev_err(ns->ctrl->device, "OCSSD revision not supported (%d)\n",
466 id->ver_id);
467 ret = -EINVAL;
468 }
469
470out:
471 kfree(id);
472 return ret;
473}
474
475static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
476 u8 *blks)
477{
478 struct request_queue *q = nvmdev->q;
479 struct nvm_geo *geo = &nvmdev->geo;
480 struct nvme_ns *ns = q->queuedata;
481 struct nvme_ctrl *ctrl = ns->ctrl;
482 struct nvme_nvm_command c = {};
483 struct nvme_nvm_bb_tbl *bb_tbl;
484 int nr_blks = geo->num_chk * geo->num_pln;
485 int tblsz = sizeof(struct nvme_nvm_bb_tbl) + nr_blks;
486 int ret = 0;
487
488 c.get_bb.opcode = nvme_nvm_admin_get_bb_tbl;
489 c.get_bb.nsid = cpu_to_le32(ns->head->ns_id);
490 c.get_bb.spba = cpu_to_le64(ppa.ppa);
491
492 bb_tbl = kzalloc(tblsz, GFP_KERNEL);
493 if (!bb_tbl)
494 return -ENOMEM;
495
496 ret = nvme_submit_sync_cmd(ctrl->admin_q, (struct nvme_command *)&c,
497 bb_tbl, tblsz);
498 if (ret) {
499 dev_err(ctrl->device, "get bad block table failed (%d)\n", ret);
500 ret = -EIO;
501 goto out;
502 }
503
504 if (bb_tbl->tblid[0] != 'B' || bb_tbl->tblid[1] != 'B' ||
505 bb_tbl->tblid[2] != 'L' || bb_tbl->tblid[3] != 'T') {
506 dev_err(ctrl->device, "bbt format mismatch\n");
507 ret = -EINVAL;
508 goto out;
509 }
510
511 if (le16_to_cpu(bb_tbl->verid) != 1) {
512 ret = -EINVAL;
513 dev_err(ctrl->device, "bbt version not supported\n");
514 goto out;
515 }
516
517 if (le32_to_cpu(bb_tbl->tblks) != nr_blks) {
518 ret = -EINVAL;
519 dev_err(ctrl->device,
520 "bbt unsuspected blocks returned (%u!=%u)",
521 le32_to_cpu(bb_tbl->tblks), nr_blks);
522 goto out;
523 }
524
525 memcpy(blks, bb_tbl->blk, geo->num_chk * geo->num_pln);
526out:
527 kfree(bb_tbl);
528 return ret;
529}
530
531static int nvme_nvm_set_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr *ppas,
532 int nr_ppas, int type)
533{
534 struct nvme_ns *ns = nvmdev->q->queuedata;
535 struct nvme_nvm_command c = {};
536 int ret = 0;
537
538 c.set_bb.opcode = nvme_nvm_admin_set_bb_tbl;
539 c.set_bb.nsid = cpu_to_le32(ns->head->ns_id);
540 c.set_bb.spba = cpu_to_le64(ppas->ppa);
541 c.set_bb.nlb = cpu_to_le16(nr_ppas - 1);
542 c.set_bb.value = type;
543
544 ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, (struct nvme_command *)&c,
545 NULL, 0);
546 if (ret)
547 dev_err(ns->ctrl->device, "set bad block table failed (%d)\n",
548 ret);
549 return ret;
550}
551
552
553
554
555static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev,
556 struct nvm_chk_meta *meta,
557 sector_t slba, int nchks)
558{
559 struct nvm_geo *geo = &ndev->geo;
560 struct nvme_ns *ns = ndev->q->queuedata;
561 struct nvme_ctrl *ctrl = ns->ctrl;
562 struct nvme_nvm_chk_meta *dev_meta = (struct nvme_nvm_chk_meta *)meta;
563 struct ppa_addr ppa;
564 size_t left = nchks * sizeof(struct nvme_nvm_chk_meta);
565 size_t log_pos, offset, len;
566 int ret, i, max_len;
567
568
569
570
571
572 max_len = min_t(unsigned int, ctrl->max_hw_sectors << 9, 256 * 1024);
573
574
575 ppa.ppa = slba;
576 ppa = dev_to_generic_addr(ndev, ppa);
577
578 log_pos = ppa.m.chk;
579 log_pos += ppa.m.pu * geo->num_chk;
580 log_pos += ppa.m.grp * geo->num_lun * geo->num_chk;
581
582 offset = log_pos * sizeof(struct nvme_nvm_chk_meta);
583
584 while (left) {
585 len = min_t(unsigned int, left, max_len);
586
587 ret = nvme_get_log(ctrl, ns->head->ns_id,
588 NVME_NVM_LOG_REPORT_CHUNK, 0, dev_meta, len,
589 offset);
590 if (ret) {
591 dev_err(ctrl->device, "Get REPORT CHUNK log error\n");
592 break;
593 }
594
595 for (i = 0; i < len; i += sizeof(struct nvme_nvm_chk_meta)) {
596 meta->state = dev_meta->state;
597 meta->type = dev_meta->type;
598 meta->wi = dev_meta->wi;
599 meta->slba = le64_to_cpu(dev_meta->slba);
600 meta->cnlb = le64_to_cpu(dev_meta->cnlb);
601 meta->wp = le64_to_cpu(dev_meta->wp);
602
603 meta++;
604 dev_meta++;
605 }
606
607 offset += len;
608 left -= len;
609 }
610
611 return ret;
612}
613
614static inline void nvme_nvm_rqtocmd(struct nvm_rq *rqd, struct nvme_ns *ns,
615 struct nvme_nvm_command *c)
616{
617 c->ph_rw.opcode = rqd->opcode;
618 c->ph_rw.nsid = cpu_to_le32(ns->head->ns_id);
619 c->ph_rw.spba = cpu_to_le64(rqd->ppa_addr.ppa);
620 c->ph_rw.metadata = cpu_to_le64(rqd->dma_meta_list);
621 c->ph_rw.control = cpu_to_le16(rqd->flags);
622 c->ph_rw.length = cpu_to_le16(rqd->nr_ppas - 1);
623}
624
625static void nvme_nvm_end_io(struct request *rq, blk_status_t status)
626{
627 struct nvm_rq *rqd = rq->end_io_data;
628
629 rqd->ppa_status = le64_to_cpu(nvme_req(rq)->result.u64);
630 rqd->error = nvme_req(rq)->status;
631 nvm_end_io(rqd);
632
633 kfree(nvme_req(rq)->cmd);
634 blk_mq_free_request(rq);
635}
636
637static struct request *nvme_nvm_alloc_request(struct request_queue *q,
638 struct nvm_rq *rqd,
639 struct nvme_nvm_command *cmd)
640{
641 struct nvme_ns *ns = q->queuedata;
642 struct request *rq;
643
644 nvme_nvm_rqtocmd(rqd, ns, cmd);
645
646 rq = nvme_alloc_request(q, (struct nvme_command *)cmd, 0, NVME_QID_ANY);
647 if (IS_ERR(rq))
648 return rq;
649
650 rq->cmd_flags &= ~REQ_FAILFAST_DRIVER;
651
652 if (rqd->bio)
653 blk_rq_append_bio(rq, &rqd->bio);
654 else
655 rq->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, IOPRIO_NORM);
656
657 return rq;
658}
659
660static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
661{
662 struct request_queue *q = dev->q;
663 struct nvme_nvm_command *cmd;
664 struct request *rq;
665
666 cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL);
667 if (!cmd)
668 return -ENOMEM;
669
670 rq = nvme_nvm_alloc_request(q, rqd, cmd);
671 if (IS_ERR(rq)) {
672 kfree(cmd);
673 return PTR_ERR(rq);
674 }
675
676 rq->end_io_data = rqd;
677
678 blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);
679
680 return 0;
681}
682
683static void *nvme_nvm_create_dma_pool(struct nvm_dev *nvmdev, char *name)
684{
685 struct nvme_ns *ns = nvmdev->q->queuedata;
686
687 return dma_pool_create(name, ns->ctrl->dev, PAGE_SIZE, PAGE_SIZE, 0);
688}
689
690static void nvme_nvm_destroy_dma_pool(void *pool)
691{
692 struct dma_pool *dma_pool = pool;
693
694 dma_pool_destroy(dma_pool);
695}
696
697static void *nvme_nvm_dev_dma_alloc(struct nvm_dev *dev, void *pool,
698 gfp_t mem_flags, dma_addr_t *dma_handler)
699{
700 return dma_pool_alloc(pool, mem_flags, dma_handler);
701}
702
703static void nvme_nvm_dev_dma_free(void *pool, void *addr,
704 dma_addr_t dma_handler)
705{
706 dma_pool_free(pool, addr, dma_handler);
707}
708
709static struct nvm_dev_ops nvme_nvm_dev_ops = {
710 .identity = nvme_nvm_identity,
711
712 .get_bb_tbl = nvme_nvm_get_bb_tbl,
713 .set_bb_tbl = nvme_nvm_set_bb_tbl,
714
715 .get_chk_meta = nvme_nvm_get_chk_meta,
716
717 .submit_io = nvme_nvm_submit_io,
718
719 .create_dma_pool = nvme_nvm_create_dma_pool,
720 .destroy_dma_pool = nvme_nvm_destroy_dma_pool,
721 .dev_dma_alloc = nvme_nvm_dev_dma_alloc,
722 .dev_dma_free = nvme_nvm_dev_dma_free,
723};
724
725static int nvme_nvm_submit_user_cmd(struct request_queue *q,
726 struct nvme_ns *ns,
727 struct nvme_nvm_command *vcmd,
728 void __user *ubuf, unsigned int bufflen,
729 void __user *meta_buf, unsigned int meta_len,
730 void __user *ppa_buf, unsigned int ppa_len,
731 u32 *result, u64 *status, unsigned int timeout)
732{
733 bool write = nvme_is_write((struct nvme_command *)vcmd);
734 struct nvm_dev *dev = ns->ndev;
735 struct gendisk *disk = ns->disk;
736 struct request *rq;
737 struct bio *bio = NULL;
738 __le64 *ppa_list = NULL;
739 dma_addr_t ppa_dma;
740 __le64 *metadata = NULL;
741 dma_addr_t metadata_dma;
742 DECLARE_COMPLETION_ONSTACK(wait);
743 int ret = 0;
744
745 rq = nvme_alloc_request(q, (struct nvme_command *)vcmd, 0,
746 NVME_QID_ANY);
747 if (IS_ERR(rq)) {
748 ret = -ENOMEM;
749 goto err_cmd;
750 }
751
752 rq->timeout = timeout ? timeout : ADMIN_TIMEOUT;
753
754 if (ppa_buf && ppa_len) {
755 ppa_list = dma_pool_alloc(dev->dma_pool, GFP_KERNEL, &ppa_dma);
756 if (!ppa_list) {
757 ret = -ENOMEM;
758 goto err_rq;
759 }
760 if (copy_from_user(ppa_list, (void __user *)ppa_buf,
761 sizeof(u64) * (ppa_len + 1))) {
762 ret = -EFAULT;
763 goto err_ppa;
764 }
765 vcmd->ph_rw.spba = cpu_to_le64(ppa_dma);
766 } else {
767 vcmd->ph_rw.spba = cpu_to_le64((uintptr_t)ppa_buf);
768 }
769
770 if (ubuf && bufflen) {
771 ret = blk_rq_map_user(q, rq, NULL, ubuf, bufflen, GFP_KERNEL);
772 if (ret)
773 goto err_ppa;
774 bio = rq->bio;
775
776 if (meta_buf && meta_len) {
777 metadata = dma_pool_alloc(dev->dma_pool, GFP_KERNEL,
778 &metadata_dma);
779 if (!metadata) {
780 ret = -ENOMEM;
781 goto err_map;
782 }
783
784 if (write) {
785 if (copy_from_user(metadata,
786 (void __user *)meta_buf,
787 meta_len)) {
788 ret = -EFAULT;
789 goto err_meta;
790 }
791 }
792 vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma);
793 }
794
795 bio->bi_disk = disk;
796 }
797
798 blk_execute_rq(q, NULL, rq, 0);
799
800 if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
801 ret = -EINTR;
802 else if (nvme_req(rq)->status & 0x7ff)
803 ret = -EIO;
804 if (result)
805 *result = nvme_req(rq)->status & 0x7ff;
806 if (status)
807 *status = le64_to_cpu(nvme_req(rq)->result.u64);
808
809 if (metadata && !ret && !write) {
810 if (copy_to_user(meta_buf, (void *)metadata, meta_len))
811 ret = -EFAULT;
812 }
813err_meta:
814 if (meta_buf && meta_len)
815 dma_pool_free(dev->dma_pool, metadata, metadata_dma);
816err_map:
817 if (bio)
818 blk_rq_unmap_user(bio);
819err_ppa:
820 if (ppa_buf && ppa_len)
821 dma_pool_free(dev->dma_pool, ppa_list, ppa_dma);
822err_rq:
823 blk_mq_free_request(rq);
824err_cmd:
825 return ret;
826}
827
828static int nvme_nvm_submit_vio(struct nvme_ns *ns,
829 struct nvm_user_vio __user *uvio)
830{
831 struct nvm_user_vio vio;
832 struct nvme_nvm_command c;
833 unsigned int length;
834 int ret;
835
836 if (copy_from_user(&vio, uvio, sizeof(vio)))
837 return -EFAULT;
838 if (vio.flags)
839 return -EINVAL;
840
841 memset(&c, 0, sizeof(c));
842 c.ph_rw.opcode = vio.opcode;
843 c.ph_rw.nsid = cpu_to_le32(ns->head->ns_id);
844 c.ph_rw.control = cpu_to_le16(vio.control);
845 c.ph_rw.length = cpu_to_le16(vio.nppas);
846
847 length = (vio.nppas + 1) << ns->lba_shift;
848
849 ret = nvme_nvm_submit_user_cmd(ns->queue, ns, &c,
850 (void __user *)(uintptr_t)vio.addr, length,
851 (void __user *)(uintptr_t)vio.metadata,
852 vio.metadata_len,
853 (void __user *)(uintptr_t)vio.ppa_list, vio.nppas,
854 &vio.result, &vio.status, 0);
855
856 if (ret && copy_to_user(uvio, &vio, sizeof(vio)))
857 return -EFAULT;
858
859 return ret;
860}
861
862static int nvme_nvm_user_vcmd(struct nvme_ns *ns, int admin,
863 struct nvm_passthru_vio __user *uvcmd)
864{
865 struct nvm_passthru_vio vcmd;
866 struct nvme_nvm_command c;
867 struct request_queue *q;
868 unsigned int timeout = 0;
869 int ret;
870
871 if (copy_from_user(&vcmd, uvcmd, sizeof(vcmd)))
872 return -EFAULT;
873 if ((vcmd.opcode != 0xF2) && (!capable(CAP_SYS_ADMIN)))
874 return -EACCES;
875 if (vcmd.flags)
876 return -EINVAL;
877
878 memset(&c, 0, sizeof(c));
879 c.common.opcode = vcmd.opcode;
880 c.common.nsid = cpu_to_le32(ns->head->ns_id);
881 c.common.cdw2[0] = cpu_to_le32(vcmd.cdw2);
882 c.common.cdw2[1] = cpu_to_le32(vcmd.cdw3);
883
884 c.ph_rw.length = cpu_to_le16(vcmd.nppas);
885 c.ph_rw.control = cpu_to_le16(vcmd.control);
886 c.common.cdw13 = cpu_to_le32(vcmd.cdw13);
887 c.common.cdw14 = cpu_to_le32(vcmd.cdw14);
888 c.common.cdw15 = cpu_to_le32(vcmd.cdw15);
889
890 if (vcmd.timeout_ms)
891 timeout = msecs_to_jiffies(vcmd.timeout_ms);
892
893 q = admin ? ns->ctrl->admin_q : ns->queue;
894
895 ret = nvme_nvm_submit_user_cmd(q, ns,
896 (struct nvme_nvm_command *)&c,
897 (void __user *)(uintptr_t)vcmd.addr, vcmd.data_len,
898 (void __user *)(uintptr_t)vcmd.metadata,
899 vcmd.metadata_len,
900 (void __user *)(uintptr_t)vcmd.ppa_list, vcmd.nppas,
901 &vcmd.result, &vcmd.status, timeout);
902
903 if (ret && copy_to_user(uvcmd, &vcmd, sizeof(vcmd)))
904 return -EFAULT;
905
906 return ret;
907}
908
909int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg)
910{
911 switch (cmd) {
912 case NVME_NVM_IOCTL_ADMIN_VIO:
913 return nvme_nvm_user_vcmd(ns, 1, (void __user *)arg);
914 case NVME_NVM_IOCTL_IO_VIO:
915 return nvme_nvm_user_vcmd(ns, 0, (void __user *)arg);
916 case NVME_NVM_IOCTL_SUBMIT_VIO:
917 return nvme_nvm_submit_vio(ns, (void __user *)arg);
918 default:
919 return -ENOTTY;
920 }
921}
922
923int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node)
924{
925 struct request_queue *q = ns->queue;
926 struct nvm_dev *dev;
927 struct nvm_geo *geo;
928
929 _nvme_nvm_check_size();
930
931 dev = nvm_alloc_dev(node);
932 if (!dev)
933 return -ENOMEM;
934
935
936 geo = &dev->geo;
937 geo->csecs = 1 << ns->lba_shift;
938 geo->sos = ns->ms;
939
940 dev->q = q;
941 memcpy(dev->name, disk_name, DISK_NAME_LEN);
942 dev->ops = &nvme_nvm_dev_ops;
943 dev->private_data = ns;
944 ns->ndev = dev;
945
946 return nvm_register(dev);
947}
948
949void nvme_nvm_unregister(struct nvme_ns *ns)
950{
951 nvm_unregister(ns->ndev);
952}
953
954static ssize_t nvm_dev_attr_show(struct device *dev,
955 struct device_attribute *dattr, char *page)
956{
957 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
958 struct nvm_dev *ndev = ns->ndev;
959 struct nvm_geo *geo = &ndev->geo;
960 struct attribute *attr;
961
962 if (!ndev)
963 return 0;
964
965 attr = &dattr->attr;
966
967 if (strcmp(attr->name, "version") == 0) {
968 if (geo->major_ver_id == 1)
969 return scnprintf(page, PAGE_SIZE, "%u\n",
970 geo->major_ver_id);
971 else
972 return scnprintf(page, PAGE_SIZE, "%u.%u\n",
973 geo->major_ver_id,
974 geo->minor_ver_id);
975 } else if (strcmp(attr->name, "capabilities") == 0) {
976 return scnprintf(page, PAGE_SIZE, "%u\n", geo->cap);
977 } else if (strcmp(attr->name, "read_typ") == 0) {
978 return scnprintf(page, PAGE_SIZE, "%u\n", geo->trdt);
979 } else if (strcmp(attr->name, "read_max") == 0) {
980 return scnprintf(page, PAGE_SIZE, "%u\n", geo->trdm);
981 } else {
982 return scnprintf(page,
983 PAGE_SIZE,
984 "Unhandled attr(%s) in `%s`\n",
985 attr->name, __func__);
986 }
987}
988
989static ssize_t nvm_dev_attr_show_ppaf(struct nvm_addrf_12 *ppaf, char *page)
990{
991 return scnprintf(page, PAGE_SIZE,
992 "0x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
993 ppaf->ch_offset, ppaf->ch_len,
994 ppaf->lun_offset, ppaf->lun_len,
995 ppaf->pln_offset, ppaf->pln_len,
996 ppaf->blk_offset, ppaf->blk_len,
997 ppaf->pg_offset, ppaf->pg_len,
998 ppaf->sec_offset, ppaf->sec_len);
999}
1000
1001static ssize_t nvm_dev_attr_show_12(struct device *dev,
1002 struct device_attribute *dattr, char *page)
1003{
1004 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1005 struct nvm_dev *ndev = ns->ndev;
1006 struct nvm_geo *geo = &ndev->geo;
1007 struct attribute *attr;
1008
1009 if (!ndev)
1010 return 0;
1011
1012 attr = &dattr->attr;
1013
1014 if (strcmp(attr->name, "vendor_opcode") == 0) {
1015 return scnprintf(page, PAGE_SIZE, "%u\n", geo->vmnt);
1016 } else if (strcmp(attr->name, "device_mode") == 0) {
1017 return scnprintf(page, PAGE_SIZE, "%u\n", geo->dom);
1018
1019 } else if (strcmp(attr->name, "media_manager") == 0) {
1020 return scnprintf(page, PAGE_SIZE, "%s\n", "gennvm");
1021 } else if (strcmp(attr->name, "ppa_format") == 0) {
1022 return nvm_dev_attr_show_ppaf((void *)&geo->addrf, page);
1023 } else if (strcmp(attr->name, "media_type") == 0) {
1024 return scnprintf(page, PAGE_SIZE, "%u\n", geo->mtype);
1025 } else if (strcmp(attr->name, "flash_media_type") == 0) {
1026 return scnprintf(page, PAGE_SIZE, "%u\n", geo->fmtype);
1027 } else if (strcmp(attr->name, "num_channels") == 0) {
1028 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_ch);
1029 } else if (strcmp(attr->name, "num_luns") == 0) {
1030 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_lun);
1031 } else if (strcmp(attr->name, "num_planes") == 0) {
1032 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_pln);
1033 } else if (strcmp(attr->name, "num_blocks") == 0) {
1034 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_chk);
1035 } else if (strcmp(attr->name, "num_pages") == 0) {
1036 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_pg);
1037 } else if (strcmp(attr->name, "page_size") == 0) {
1038 return scnprintf(page, PAGE_SIZE, "%u\n", geo->fpg_sz);
1039 } else if (strcmp(attr->name, "hw_sector_size") == 0) {
1040 return scnprintf(page, PAGE_SIZE, "%u\n", geo->csecs);
1041 } else if (strcmp(attr->name, "oob_sector_size") == 0) {
1042 return scnprintf(page, PAGE_SIZE, "%u\n", geo->sos);
1043 } else if (strcmp(attr->name, "prog_typ") == 0) {
1044 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tprt);
1045 } else if (strcmp(attr->name, "prog_max") == 0) {
1046 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tprm);
1047 } else if (strcmp(attr->name, "erase_typ") == 0) {
1048 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tbet);
1049 } else if (strcmp(attr->name, "erase_max") == 0) {
1050 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tbem);
1051 } else if (strcmp(attr->name, "multiplane_modes") == 0) {
1052 return scnprintf(page, PAGE_SIZE, "0x%08x\n", geo->mpos);
1053 } else if (strcmp(attr->name, "media_capabilities") == 0) {
1054 return scnprintf(page, PAGE_SIZE, "0x%08x\n", geo->mccap);
1055 } else if (strcmp(attr->name, "max_phys_secs") == 0) {
1056 return scnprintf(page, PAGE_SIZE, "%u\n", NVM_MAX_VLBA);
1057 } else {
1058 return scnprintf(page, PAGE_SIZE,
1059 "Unhandled attr(%s) in `%s`\n",
1060 attr->name, __func__);
1061 }
1062}
1063
1064static ssize_t nvm_dev_attr_show_20(struct device *dev,
1065 struct device_attribute *dattr, char *page)
1066{
1067 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1068 struct nvm_dev *ndev = ns->ndev;
1069 struct nvm_geo *geo = &ndev->geo;
1070 struct attribute *attr;
1071
1072 if (!ndev)
1073 return 0;
1074
1075 attr = &dattr->attr;
1076
1077 if (strcmp(attr->name, "groups") == 0) {
1078 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_ch);
1079 } else if (strcmp(attr->name, "punits") == 0) {
1080 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_lun);
1081 } else if (strcmp(attr->name, "chunks") == 0) {
1082 return scnprintf(page, PAGE_SIZE, "%u\n", geo->num_chk);
1083 } else if (strcmp(attr->name, "clba") == 0) {
1084 return scnprintf(page, PAGE_SIZE, "%u\n", geo->clba);
1085 } else if (strcmp(attr->name, "ws_min") == 0) {
1086 return scnprintf(page, PAGE_SIZE, "%u\n", geo->ws_min);
1087 } else if (strcmp(attr->name, "ws_opt") == 0) {
1088 return scnprintf(page, PAGE_SIZE, "%u\n", geo->ws_opt);
1089 } else if (strcmp(attr->name, "maxoc") == 0) {
1090 return scnprintf(page, PAGE_SIZE, "%u\n", geo->maxoc);
1091 } else if (strcmp(attr->name, "maxocpu") == 0) {
1092 return scnprintf(page, PAGE_SIZE, "%u\n", geo->maxocpu);
1093 } else if (strcmp(attr->name, "mw_cunits") == 0) {
1094 return scnprintf(page, PAGE_SIZE, "%u\n", geo->mw_cunits);
1095 } else if (strcmp(attr->name, "write_typ") == 0) {
1096 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tprt);
1097 } else if (strcmp(attr->name, "write_max") == 0) {
1098 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tprm);
1099 } else if (strcmp(attr->name, "reset_typ") == 0) {
1100 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tbet);
1101 } else if (strcmp(attr->name, "reset_max") == 0) {
1102 return scnprintf(page, PAGE_SIZE, "%u\n", geo->tbem);
1103 } else {
1104 return scnprintf(page, PAGE_SIZE,
1105 "Unhandled attr(%s) in `%s`\n",
1106 attr->name, __func__);
1107 }
1108}
1109
1110#define NVM_DEV_ATTR_RO(_name) \
1111 DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show, NULL)
1112#define NVM_DEV_ATTR_12_RO(_name) \
1113 DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show_12, NULL)
1114#define NVM_DEV_ATTR_20_RO(_name) \
1115 DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show_20, NULL)
1116
1117
1118static NVM_DEV_ATTR_RO(version);
1119static NVM_DEV_ATTR_RO(capabilities);
1120
1121static NVM_DEV_ATTR_RO(read_typ);
1122static NVM_DEV_ATTR_RO(read_max);
1123
1124
1125static NVM_DEV_ATTR_12_RO(vendor_opcode);
1126static NVM_DEV_ATTR_12_RO(device_mode);
1127static NVM_DEV_ATTR_12_RO(ppa_format);
1128static NVM_DEV_ATTR_12_RO(media_manager);
1129static NVM_DEV_ATTR_12_RO(media_type);
1130static NVM_DEV_ATTR_12_RO(flash_media_type);
1131static NVM_DEV_ATTR_12_RO(num_channels);
1132static NVM_DEV_ATTR_12_RO(num_luns);
1133static NVM_DEV_ATTR_12_RO(num_planes);
1134static NVM_DEV_ATTR_12_RO(num_blocks);
1135static NVM_DEV_ATTR_12_RO(num_pages);
1136static NVM_DEV_ATTR_12_RO(page_size);
1137static NVM_DEV_ATTR_12_RO(hw_sector_size);
1138static NVM_DEV_ATTR_12_RO(oob_sector_size);
1139static NVM_DEV_ATTR_12_RO(prog_typ);
1140static NVM_DEV_ATTR_12_RO(prog_max);
1141static NVM_DEV_ATTR_12_RO(erase_typ);
1142static NVM_DEV_ATTR_12_RO(erase_max);
1143static NVM_DEV_ATTR_12_RO(multiplane_modes);
1144static NVM_DEV_ATTR_12_RO(media_capabilities);
1145static NVM_DEV_ATTR_12_RO(max_phys_secs);
1146
1147
1148static NVM_DEV_ATTR_20_RO(groups);
1149static NVM_DEV_ATTR_20_RO(punits);
1150static NVM_DEV_ATTR_20_RO(chunks);
1151static NVM_DEV_ATTR_20_RO(clba);
1152static NVM_DEV_ATTR_20_RO(ws_min);
1153static NVM_DEV_ATTR_20_RO(ws_opt);
1154static NVM_DEV_ATTR_20_RO(maxoc);
1155static NVM_DEV_ATTR_20_RO(maxocpu);
1156static NVM_DEV_ATTR_20_RO(mw_cunits);
1157static NVM_DEV_ATTR_20_RO(write_typ);
1158static NVM_DEV_ATTR_20_RO(write_max);
1159static NVM_DEV_ATTR_20_RO(reset_typ);
1160static NVM_DEV_ATTR_20_RO(reset_max);
1161
1162static struct attribute *nvm_dev_attrs[] = {
1163
1164 &dev_attr_version.attr,
1165 &dev_attr_capabilities.attr,
1166 &dev_attr_read_typ.attr,
1167 &dev_attr_read_max.attr,
1168
1169
1170 &dev_attr_vendor_opcode.attr,
1171 &dev_attr_device_mode.attr,
1172 &dev_attr_media_manager.attr,
1173 &dev_attr_ppa_format.attr,
1174 &dev_attr_media_type.attr,
1175 &dev_attr_flash_media_type.attr,
1176 &dev_attr_num_channels.attr,
1177 &dev_attr_num_luns.attr,
1178 &dev_attr_num_planes.attr,
1179 &dev_attr_num_blocks.attr,
1180 &dev_attr_num_pages.attr,
1181 &dev_attr_page_size.attr,
1182 &dev_attr_hw_sector_size.attr,
1183 &dev_attr_oob_sector_size.attr,
1184 &dev_attr_prog_typ.attr,
1185 &dev_attr_prog_max.attr,
1186 &dev_attr_erase_typ.attr,
1187 &dev_attr_erase_max.attr,
1188 &dev_attr_multiplane_modes.attr,
1189 &dev_attr_media_capabilities.attr,
1190 &dev_attr_max_phys_secs.attr,
1191
1192
1193 &dev_attr_groups.attr,
1194 &dev_attr_punits.attr,
1195 &dev_attr_chunks.attr,
1196 &dev_attr_clba.attr,
1197 &dev_attr_ws_min.attr,
1198 &dev_attr_ws_opt.attr,
1199 &dev_attr_maxoc.attr,
1200 &dev_attr_maxocpu.attr,
1201 &dev_attr_mw_cunits.attr,
1202
1203 &dev_attr_write_typ.attr,
1204 &dev_attr_write_max.attr,
1205 &dev_attr_reset_typ.attr,
1206 &dev_attr_reset_max.attr,
1207
1208 NULL,
1209};
1210
1211static umode_t nvm_dev_attrs_visible(struct kobject *kobj,
1212 struct attribute *attr, int index)
1213{
1214 struct device *dev = container_of(kobj, struct device, kobj);
1215 struct gendisk *disk = dev_to_disk(dev);
1216 struct nvme_ns *ns = disk->private_data;
1217 struct nvm_dev *ndev = ns->ndev;
1218 struct device_attribute *dev_attr =
1219 container_of(attr, typeof(*dev_attr), attr);
1220
1221 if (!ndev)
1222 return 0;
1223
1224 if (dev_attr->show == nvm_dev_attr_show)
1225 return attr->mode;
1226
1227 switch (ndev->geo.major_ver_id) {
1228 case 1:
1229 if (dev_attr->show == nvm_dev_attr_show_12)
1230 return attr->mode;
1231 break;
1232 case 2:
1233 if (dev_attr->show == nvm_dev_attr_show_20)
1234 return attr->mode;
1235 break;
1236 }
1237
1238 return 0;
1239}
1240
1241const struct attribute_group nvme_nvm_attr_group = {
1242 .name = "lightnvm",
1243 .attrs = nvm_dev_attrs,
1244 .is_visible = nvm_dev_attrs_visible,
1245};
1246
