1
2
3
4
5
6#include <linux/ptrace.h>
7#include <linux/nvme_ioctl.h>
8#include "nvme.h"
9
10
11
12
13
14
15static void __user *nvme_to_user_ptr(uintptr_t ptrval)
16{
17 if (in_compat_syscall())
18 ptrval = (compat_uptr_t)ptrval;
19 return (void __user *)ptrval;
20}
21
22static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
23 unsigned len, u32 seed, bool write)
24{
25 struct bio_integrity_payload *bip;
26 int ret = -ENOMEM;
27 void *buf;
28
29 buf = kmalloc(len, GFP_KERNEL);
30 if (!buf)
31 goto out;
32
33 ret = -EFAULT;
34 if (write && copy_from_user(buf, ubuf, len))
35 goto out_free_meta;
36
37 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
38 if (IS_ERR(bip)) {
39 ret = PTR_ERR(bip);
40 goto out_free_meta;
41 }
42
43 bip->bip_iter.bi_size = len;
44 bip->bip_iter.bi_sector = seed;
45 ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
46 offset_in_page(buf));
47 if (ret == len)
48 return buf;
49 ret = -ENOMEM;
50out_free_meta:
51 kfree(buf);
52out:
53 return ERR_PTR(ret);
54}
55
56static int nvme_submit_user_cmd(struct request_queue *q,
57 struct nvme_command *cmd, void __user *ubuffer,
58 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
59 u32 meta_seed, u64 *result, unsigned timeout, bool vec)
60{
61 bool write = nvme_is_write(cmd);
62 struct nvme_ns *ns = q->queuedata;
63 struct block_device *bdev = ns ? ns->disk->part0 : NULL;
64 struct request *req;
65 struct bio *bio = NULL;
66 void *meta = NULL;
67 int ret;
68
69 req = blk_mq_alloc_request(q, nvme_req_op(cmd), 0);
70 if (IS_ERR(req))
71 return PTR_ERR(req);
72 nvme_init_request(req, cmd);
73
74 if (timeout)
75 req->timeout = timeout;
76 nvme_req(req)->flags |= NVME_REQ_USERCMD;
77
78 if (ubuffer && bufflen) {
79 if (!vec)
80 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
81 GFP_KERNEL);
82 else {
83 struct iovec fast_iov[UIO_FASTIOV];
84 struct iovec *iov = fast_iov;
85 struct iov_iter iter;
86
87 ret = import_iovec(rq_data_dir(req), ubuffer, bufflen,
88 UIO_FASTIOV, &iov, &iter);
89 if (ret < 0)
90 goto out;
91 ret = blk_rq_map_user_iov(q, req, NULL, &iter,
92 GFP_KERNEL);
93 kfree(iov);
94 }
95 if (ret)
96 goto out;
97 bio = req->bio;
98 if (bdev)
99 bio_set_dev(bio, bdev);
100 if (bdev && meta_buffer && meta_len) {
101 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
102 meta_seed, write);
103 if (IS_ERR(meta)) {
104 ret = PTR_ERR(meta);
105 goto out_unmap;
106 }
107 req->cmd_flags |= REQ_INTEGRITY;
108 }
109 }
110
111 ret = nvme_execute_passthru_rq(req);
112 if (result)
113 *result = le64_to_cpu(nvme_req(req)->result.u64);
114 if (meta && !ret && !write) {
115 if (copy_to_user(meta_buffer, meta, meta_len))
116 ret = -EFAULT;
117 }
118 kfree(meta);
119 out_unmap:
120 if (bio)
121 blk_rq_unmap_user(bio);
122 out:
123 blk_mq_free_request(req);
124 return ret;
125}
126
127
128static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
129{
130 struct nvme_user_io io;
131 struct nvme_command c;
132 unsigned length, meta_len;
133 void __user *metadata;
134
135 if (copy_from_user(&io, uio, sizeof(io)))
136 return -EFAULT;
137 if (io.flags)
138 return -EINVAL;
139
140 switch (io.opcode) {
141 case nvme_cmd_write:
142 case nvme_cmd_read:
143 case nvme_cmd_compare:
144 break;
145 default:
146 return -EINVAL;
147 }
148
149 length = (io.nblocks + 1) << ns->lba_shift;
150
151 if ((io.control & NVME_RW_PRINFO_PRACT) &&
152 ns->ms == sizeof(struct t10_pi_tuple)) {
153
154
155
156
157 if (nvme_to_user_ptr(io.metadata))
158 return -EINVAL;
159 meta_len = 0;
160 metadata = NULL;
161 } else {
162 meta_len = (io.nblocks + 1) * ns->ms;
163 metadata = nvme_to_user_ptr(io.metadata);
164 }
165
166 if (ns->features & NVME_NS_EXT_LBAS) {
167 length += meta_len;
168 meta_len = 0;
169 } else if (meta_len) {
170 if ((io.metadata & 3) || !io.metadata)
171 return -EINVAL;
172 }
173
174 memset(&c, 0, sizeof(c));
175 c.rw.opcode = io.opcode;
176 c.rw.flags = io.flags;
177 c.rw.nsid = cpu_to_le32(ns->head->ns_id);
178 c.rw.slba = cpu_to_le64(io.slba);
179 c.rw.length = cpu_to_le16(io.nblocks);
180 c.rw.control = cpu_to_le16(io.control);
181 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
182 c.rw.reftag = cpu_to_le32(io.reftag);
183 c.rw.apptag = cpu_to_le16(io.apptag);
184 c.rw.appmask = cpu_to_le16(io.appmask);
185
186 return nvme_submit_user_cmd(ns->queue, &c,
187 nvme_to_user_ptr(io.addr), length,
188 metadata, meta_len, lower_32_bits(io.slba), NULL, 0,
189 false);
190}
191
192static bool nvme_validate_passthru_nsid(struct nvme_ctrl *ctrl,
193 struct nvme_ns *ns, __u32 nsid)
194{
195 if (ns && nsid != ns->head->ns_id) {
196 dev_err(ctrl->device,
197 "%s: nsid (%u) in cmd does not match nsid (%u)"
198 "of namespace\n",
199 current->comm, nsid, ns->head->ns_id);
200 return false;
201 }
202
203 return true;
204}
205
206static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
207 struct nvme_passthru_cmd __user *ucmd)
208{
209 struct nvme_passthru_cmd cmd;
210 struct nvme_command c;
211 unsigned timeout = 0;
212 u64 result;
213 int status;
214
215 if (!capable(CAP_SYS_ADMIN))
216 return -EACCES;
217 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
218 return -EFAULT;
219 if (cmd.flags)
220 return -EINVAL;
221 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))
222 return -EINVAL;
223
224 memset(&c, 0, sizeof(c));
225 c.common.opcode = cmd.opcode;
226 c.common.flags = cmd.flags;
227 c.common.nsid = cpu_to_le32(cmd.nsid);
228 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
229 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
230 c.common.cdw10 = cpu_to_le32(cmd.cdw10);
231 c.common.cdw11 = cpu_to_le32(cmd.cdw11);
232 c.common.cdw12 = cpu_to_le32(cmd.cdw12);
233 c.common.cdw13 = cpu_to_le32(cmd.cdw13);
234 c.common.cdw14 = cpu_to_le32(cmd.cdw14);
235 c.common.cdw15 = cpu_to_le32(cmd.cdw15);
236
237 if (cmd.timeout_ms)
238 timeout = msecs_to_jiffies(cmd.timeout_ms);
239
240 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
241 nvme_to_user_ptr(cmd.addr), cmd.data_len,
242 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
243 0, &result, timeout, false);
244
245 if (status >= 0) {
246 if (put_user(result, &ucmd->result))
247 return -EFAULT;
248 }
249
250 return status;
251}
252
253static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
254 struct nvme_passthru_cmd64 __user *ucmd, bool vec)
255{
256 struct nvme_passthru_cmd64 cmd;
257 struct nvme_command c;
258 unsigned timeout = 0;
259 int status;
260
261 if (!capable(CAP_SYS_ADMIN))
262 return -EACCES;
263 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
264 return -EFAULT;
265 if (cmd.flags)
266 return -EINVAL;
267 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))
268 return -EINVAL;
269
270 memset(&c, 0, sizeof(c));
271 c.common.opcode = cmd.opcode;
272 c.common.flags = cmd.flags;
273 c.common.nsid = cpu_to_le32(cmd.nsid);
274 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
275 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
276 c.common.cdw10 = cpu_to_le32(cmd.cdw10);
277 c.common.cdw11 = cpu_to_le32(cmd.cdw11);
278 c.common.cdw12 = cpu_to_le32(cmd.cdw12);
279 c.common.cdw13 = cpu_to_le32(cmd.cdw13);
280 c.common.cdw14 = cpu_to_le32(cmd.cdw14);
281 c.common.cdw15 = cpu_to_le32(cmd.cdw15);
282
283 if (cmd.timeout_ms)
284 timeout = msecs_to_jiffies(cmd.timeout_ms);
285
286 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
287 nvme_to_user_ptr(cmd.addr), cmd.data_len,
288 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
289 0, &cmd.result, timeout, vec);
290
291 if (status >= 0) {
292 if (put_user(cmd.result, &ucmd->result))
293 return -EFAULT;
294 }
295
296 return status;
297}
298
299static bool is_ctrl_ioctl(unsigned int cmd)
300{
301 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
302 return true;
303 if (is_sed_ioctl(cmd))
304 return true;
305 return false;
306}
307
308static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd,
309 void __user *argp)
310{
311 switch (cmd) {
312 case NVME_IOCTL_ADMIN_CMD:
313 return nvme_user_cmd(ctrl, NULL, argp);
314 case NVME_IOCTL_ADMIN64_CMD:
315 return nvme_user_cmd64(ctrl, NULL, argp, false);
316 default:
317 return sed_ioctl(ctrl->opal_dev, cmd, argp);
318 }
319}
320
321#ifdef COMPAT_FOR_U64_ALIGNMENT
322struct nvme_user_io32 {
323 __u8 opcode;
324 __u8 flags;
325 __u16 control;
326 __u16 nblocks;
327 __u16 rsvd;
328 __u64 metadata;
329 __u64 addr;
330 __u64 slba;
331 __u32 dsmgmt;
332 __u32 reftag;
333 __u16 apptag;
334 __u16 appmask;
335} __attribute__((__packed__));
336#define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32)
337#endif
338
339static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd,
340 void __user *argp)
341{
342 switch (cmd) {
343 case NVME_IOCTL_ID:
344 force_successful_syscall_return();
345 return ns->head->ns_id;
346 case NVME_IOCTL_IO_CMD:
347 return nvme_user_cmd(ns->ctrl, ns, argp);
348
349
350
351
352
353#ifdef COMPAT_FOR_U64_ALIGNMENT
354 case NVME_IOCTL_SUBMIT_IO32:
355#endif
356 case NVME_IOCTL_SUBMIT_IO:
357 return nvme_submit_io(ns, argp);
358 case NVME_IOCTL_IO64_CMD:
359 return nvme_user_cmd64(ns->ctrl, ns, argp, false);
360 case NVME_IOCTL_IO64_CMD_VEC:
361 return nvme_user_cmd64(ns->ctrl, ns, argp, true);
362 default:
363 return -ENOTTY;
364 }
365}
366
367static int __nvme_ioctl(struct nvme_ns *ns, unsigned int cmd, void __user *arg)
368{
369 if (is_ctrl_ioctl(cmd))
370 return nvme_ctrl_ioctl(ns->ctrl, cmd, arg);
371 return nvme_ns_ioctl(ns, cmd, arg);
372}
373
374int nvme_ioctl(struct block_device *bdev, fmode_t mode,
375 unsigned int cmd, unsigned long arg)
376{
377 struct nvme_ns *ns = bdev->bd_disk->private_data;
378
379 return __nvme_ioctl(ns, cmd, (void __user *)arg);
380}
381
382long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
383{
384 struct nvme_ns *ns =
385 container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev);
386
387 return __nvme_ioctl(ns, cmd, (void __user *)arg);
388}
389
390#ifdef CONFIG_NVME_MULTIPATH
391static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
392 void __user *argp, struct nvme_ns_head *head, int srcu_idx)
393 __releases(&head->srcu)
394{
395 struct nvme_ctrl *ctrl = ns->ctrl;
396 int ret;
397
398 nvme_get_ctrl(ns->ctrl);
399 srcu_read_unlock(&head->srcu, srcu_idx);
400 ret = nvme_ctrl_ioctl(ns->ctrl, cmd, argp);
401
402 nvme_put_ctrl(ctrl);
403 return ret;
404}
405
406int nvme_ns_head_ioctl(struct block_device *bdev, fmode_t mode,
407 unsigned int cmd, unsigned long arg)
408{
409 struct nvme_ns_head *head = bdev->bd_disk->private_data;
410 void __user *argp = (void __user *)arg;
411 struct nvme_ns *ns;
412 int srcu_idx, ret = -EWOULDBLOCK;
413
414 srcu_idx = srcu_read_lock(&head->srcu);
415 ns = nvme_find_path(head);
416 if (!ns)
417 goto out_unlock;
418
419
420
421
422
423
424 if (is_ctrl_ioctl(cmd))
425 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
426
427 ret = nvme_ns_ioctl(ns, cmd, argp);
428out_unlock:
429 srcu_read_unlock(&head->srcu, srcu_idx);
430 return ret;
431}
432
433long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
434 unsigned long arg)
435{
436 struct cdev *cdev = file_inode(file)->i_cdev;
437 struct nvme_ns_head *head =
438 container_of(cdev, struct nvme_ns_head, cdev);
439 void __user *argp = (void __user *)arg;
440 struct nvme_ns *ns;
441 int srcu_idx, ret = -EWOULDBLOCK;
442
443 srcu_idx = srcu_read_lock(&head->srcu);
444 ns = nvme_find_path(head);
445 if (!ns)
446 goto out_unlock;
447
448 if (is_ctrl_ioctl(cmd))
449 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
450
451 ret = nvme_ns_ioctl(ns, cmd, argp);
452out_unlock:
453 srcu_read_unlock(&head->srcu, srcu_idx);
454 return ret;
455}
456#endif
457
458static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
459{
460 struct nvme_ns *ns;
461 int ret;
462
463 down_read(&ctrl->namespaces_rwsem);
464 if (list_empty(&ctrl->namespaces)) {
465 ret = -ENOTTY;
466 goto out_unlock;
467 }
468
469 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
470 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
471 dev_warn(ctrl->device,
472 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
473 ret = -EINVAL;
474 goto out_unlock;
475 }
476
477 dev_warn(ctrl->device,
478 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
479 kref_get(&ns->kref);
480 up_read(&ctrl->namespaces_rwsem);
481
482 ret = nvme_user_cmd(ctrl, ns, argp);
483 nvme_put_ns(ns);
484 return ret;
485
486out_unlock:
487 up_read(&ctrl->namespaces_rwsem);
488 return ret;
489}
490
491long nvme_dev_ioctl(struct file *file, unsigned int cmd,
492 unsigned long arg)
493{
494 struct nvme_ctrl *ctrl = file->private_data;
495 void __user *argp = (void __user *)arg;
496
497 switch (cmd) {
498 case NVME_IOCTL_ADMIN_CMD:
499 return nvme_user_cmd(ctrl, NULL, argp);
500 case NVME_IOCTL_ADMIN64_CMD:
501 return nvme_user_cmd64(ctrl, NULL, argp, false);
502 case NVME_IOCTL_IO_CMD:
503 return nvme_dev_user_cmd(ctrl, argp);
504 case NVME_IOCTL_RESET:
505 dev_warn(ctrl->device, "resetting controller\n");
506 return nvme_reset_ctrl_sync(ctrl);
507 case NVME_IOCTL_SUBSYS_RESET:
508 return nvme_reset_subsystem(ctrl);
509 case NVME_IOCTL_RESCAN:
510 nvme_queue_scan(ctrl);
511 return 0;
512 default:
513 return -ENOTTY;
514 }
515}
516
