1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2011-2014, Intel Corporation. 4 * Copyright (c) 2017-2021 Christoph Hellwig. 5 */ 6 #include <linux/blk-integrity.h> 7 #include <linux/ptrace.h> /* for force_successful_syscall_return */ 8 #include <linux/nvme_ioctl.h> 9 #include <linux/io_uring/cmd.h> 10 #include "nvme.h" 11 12 enum { 13 NVME_IOCTL_VEC = (1 << 0), 14 NVME_IOCTL_PARTITION = (1 << 1), 15 }; 16 17 static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl, 18 struct nvme_command *c) 19 { 20 /* 21 * Do not allow unprivileged passthrough of admin commands except 22 * for a subset of identify commands that contain information required 23 * to form proper I/O commands in userspace and do not expose any 24 * potentially sensitive information. 25 */ 26 switch (c->common.opcode) { 27 case nvme_admin_identify: 28 switch (c->identify.cns) { 29 case NVME_ID_CNS_NS: 30 case NVME_ID_CNS_CS_NS: 31 case NVME_ID_CNS_NS_CS_INDEP: 32 case NVME_ID_CNS_CS_CTRL: 33 case NVME_ID_CNS_CTRL: 34 return true; 35 } 36 break; 37 case nvme_admin_set_features: 38 /* 39 * Reject Set Features that change controller state the driver 40 * manages itself; setting them behind the driver's back from 41 * userspace leaves it unable to react correctly. Keep Alive is 42 * only armed for fabrics - on other transports it has no 43 * reserved tag and harms idle power states. 44 */ 45 switch (le32_to_cpu(c->features.fid) & 0xff) { 46 case NVME_FEAT_KATO: 47 if (ctrl->ops->flags & NVME_F_FABRICS) 48 break; 49 fallthrough; 50 case NVME_FEAT_HOST_BEHAVIOR: 51 case NVME_FEAT_HOST_MEM_BUF: 52 case NVME_FEAT_NUM_QUEUES: 53 case NVME_FEAT_AUTO_PST: 54 return false; 55 } 56 break; 57 } 58 return capable(CAP_SYS_ADMIN); 59 } 60 61 static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, 62 bool open_for_write) 63 { 64 u32 effects; 65 66 /* 67 * Check if the controller provides a Commands Supported and Effects log 68 * and marks this command as supported. If not reject unprivileged 69 * passthrough. 70 */ 71 effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode); 72 if (!(effects & NVME_CMD_EFFECTS_CSUPP)) 73 return capable(CAP_SYS_ADMIN); 74 75 /* 76 * Don't allow passthrough for command that have intrusive (or unknown) 77 * effects. 78 */ 79 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | 80 NVME_CMD_EFFECTS_UUID_SEL | 81 NVME_CMD_EFFECTS_SCOPE_MASK)) 82 return capable(CAP_SYS_ADMIN); 83 84 /* 85 * Only allow I/O commands that transfer data to the controller or that 86 * change the logical block contents if the file descriptor is open for 87 * writing. 88 */ 89 if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) && 90 !open_for_write) 91 return capable(CAP_SYS_ADMIN); 92 93 return true; 94 } 95 96 static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 97 struct nvme_command *c, unsigned int flags, 98 bool open_for_write) 99 { 100 /* 101 * Do not allow unprivileged passthrough on partitions, as that 102 * allows an escape from the containment of the partition. 103 */ 104 if (flags & NVME_IOCTL_PARTITION) 105 return capable(CAP_SYS_ADMIN); 106 107 /* 108 * Do not allow unprivileged processes to send vendor specific or 109 * fabrics commands as we can't be sure about their effects. 110 */ 111 if (c->common.opcode >= nvme_cmd_vendor_start || 112 c->common.opcode == nvme_fabrics_command) 113 return capable(CAP_SYS_ADMIN); 114 115 if (!ns) 116 return nvme_admin_cmd_allowed(ctrl, c); 117 118 return nvme_ns_cmd_allowed(ns, c, open_for_write); 119 } 120 121 /* 122 * Convert integer values from ioctl structures to user pointers, silently 123 * ignoring the upper bits in the compat case to match behaviour of 32-bit 124 * kernels. 125 */ 126 static void __user *nvme_to_user_ptr(uintptr_t ptrval) 127 { 128 if (in_compat_syscall()) 129 ptrval = (compat_uptr_t)ptrval; 130 return (void __user *)ptrval; 131 } 132 133 static struct request *nvme_alloc_user_request(struct request_queue *q, 134 struct nvme_command *cmd, blk_opf_t rq_flags, 135 blk_mq_req_flags_t blk_flags) 136 { 137 struct nvme_ns *ns = q->queuedata; 138 struct request *req; 139 140 /* 141 * The NVME_MPATH flag is set only for IO commands sent to a namespace 142 * with a multipath enabled head. The request is not eligible for 143 * failover as passthrough requests also append REQ_FAILFAST_DRIVER. 144 */ 145 if (ns && nvme_ns_head_multipath(ns->head)) 146 rq_flags |= REQ_NVME_MPATH; 147 148 req = blk_mq_alloc_request(q, nvme_req_op(cmd) | rq_flags, blk_flags); 149 if (IS_ERR(req)) 150 return req; 151 nvme_init_request(req, cmd); 152 nvme_req(req)->flags |= NVME_REQ_USERCMD; 153 return req; 154 } 155 156 static int nvme_map_user_request(struct request *req, u64 ubuffer, 157 unsigned bufflen, void __user *meta_buffer, unsigned meta_len, 158 struct iov_iter *iter, unsigned int flags) 159 { 160 struct request_queue *q = req->q; 161 struct nvme_ns *ns = q->queuedata; 162 struct block_device *bdev = ns ? ns->disk->part0 : NULL; 163 bool supports_metadata = bdev && blk_get_integrity(bdev->bd_disk); 164 bool has_metadata = meta_buffer && meta_len; 165 int ret; 166 167 if (has_metadata && !supports_metadata) 168 return -EINVAL; 169 170 if (iter) 171 ret = blk_rq_map_user_iov(q, req, NULL, iter, GFP_KERNEL); 172 else 173 ret = blk_rq_map_user_io(req, NULL, nvme_to_user_ptr(ubuffer), 174 bufflen, GFP_KERNEL, flags & NVME_IOCTL_VEC, 0, 175 0, rq_data_dir(req)); 176 if (ret) 177 return ret; 178 179 if (has_metadata) { 180 ret = blk_rq_integrity_map_user(req, meta_buffer, meta_len); 181 if (ret) 182 goto out_unmap; 183 } 184 185 return ret; 186 187 out_unmap: 188 if (req->bio) 189 blk_rq_unmap_user(req->bio); 190 return ret; 191 } 192 193 static int nvme_submit_user_cmd(struct request_queue *q, 194 struct nvme_command *cmd, u64 ubuffer, unsigned bufflen, 195 void __user *meta_buffer, unsigned meta_len, 196 u64 *result, unsigned timeout, unsigned int flags) 197 { 198 struct nvme_ns *ns = q->queuedata; 199 struct nvme_ctrl *ctrl; 200 struct request *req; 201 struct bio *bio; 202 u32 effects; 203 int ret; 204 205 req = nvme_alloc_user_request(q, cmd, 0, 0); 206 if (IS_ERR(req)) 207 return PTR_ERR(req); 208 209 req->timeout = timeout; 210 if (ubuffer && bufflen) { 211 ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer, 212 meta_len, NULL, flags); 213 if (ret) 214 goto out_free_req; 215 } 216 217 bio = req->bio; 218 ctrl = nvme_req(req)->ctrl; 219 220 effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode); 221 ret = nvme_execute_rq(req, false); 222 if (result) 223 *result = le64_to_cpu(nvme_req(req)->result.u64); 224 if (bio) 225 blk_rq_unmap_user(bio); 226 blk_mq_free_request(req); 227 228 if (effects) 229 nvme_passthru_end(ctrl, ns, effects, cmd, ret); 230 return ret; 231 232 out_free_req: 233 blk_mq_free_request(req); 234 return ret; 235 } 236 237 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, 238 unsigned int flags, bool open_for_write) 239 { 240 struct nvme_user_io io; 241 struct nvme_command c; 242 unsigned length, meta_len; 243 void __user *metadata; 244 245 if (copy_from_user(&io, uio, sizeof(io))) 246 return -EFAULT; 247 if (io.flags) 248 return -EINVAL; 249 250 switch (io.opcode) { 251 case nvme_cmd_write: 252 case nvme_cmd_read: 253 case nvme_cmd_compare: 254 break; 255 default: 256 return -EINVAL; 257 } 258 259 length = (io.nblocks + 1) << ns->head->lba_shift; 260 261 if ((io.control & NVME_RW_PRINFO_PRACT) && 262 (ns->head->ms == ns->head->pi_size)) { 263 /* 264 * Protection information is stripped/inserted by the 265 * controller. 266 */ 267 if (nvme_to_user_ptr(io.metadata)) 268 return -EINVAL; 269 meta_len = 0; 270 metadata = NULL; 271 } else { 272 meta_len = (io.nblocks + 1) * ns->head->ms; 273 metadata = nvme_to_user_ptr(io.metadata); 274 } 275 276 if (ns->head->features & NVME_NS_EXT_LBAS) { 277 length += meta_len; 278 meta_len = 0; 279 } else if (meta_len) { 280 if ((io.metadata & 3) || !io.metadata) 281 return -EINVAL; 282 } 283 284 memset(&c, 0, sizeof(c)); 285 c.rw.opcode = io.opcode; 286 c.rw.flags = io.flags; 287 c.rw.nsid = cpu_to_le32(ns->head->ns_id); 288 c.rw.slba = cpu_to_le64(io.slba); 289 c.rw.length = cpu_to_le16(io.nblocks); 290 c.rw.control = cpu_to_le16(io.control); 291 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 292 c.rw.reftag = cpu_to_le32(io.reftag); 293 c.rw.lbat = cpu_to_le16(io.apptag); 294 c.rw.lbatm = cpu_to_le16(io.appmask); 295 296 if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write)) 297 return -EACCES; 298 299 return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, 300 meta_len, NULL, 0, 0); 301 } 302 303 static bool nvme_validate_passthru_nsid(struct nvme_ctrl *ctrl, 304 struct nvme_ns *ns, __u32 nsid) 305 { 306 if (ns && nsid != ns->head->ns_id) { 307 dev_err(ctrl->device, 308 "%s: nsid (%u) in cmd does not match nsid (%u) of namespace\n", 309 current->comm, nsid, ns->head->ns_id); 310 return false; 311 } 312 313 return true; 314 } 315 316 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 317 struct nvme_passthru_cmd __user *ucmd, unsigned int flags, 318 bool open_for_write) 319 { 320 struct nvme_passthru_cmd cmd; 321 struct nvme_command c; 322 unsigned timeout = 0; 323 u64 result; 324 int status; 325 326 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 327 return -EFAULT; 328 if (cmd.flags) 329 return -EINVAL; 330 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid)) 331 return -EINVAL; 332 333 memset(&c, 0, sizeof(c)); 334 c.common.opcode = cmd.opcode; 335 c.common.flags = cmd.flags; 336 c.common.nsid = cpu_to_le32(cmd.nsid); 337 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 338 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 339 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 340 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 341 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 342 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 343 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 344 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 345 346 if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) 347 return -EACCES; 348 349 if (cmd.timeout_ms) 350 timeout = msecs_to_jiffies(cmd.timeout_ms); 351 352 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 353 cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata), 354 cmd.metadata_len, &result, timeout, 0); 355 356 if (status >= 0) { 357 if (put_user(result, &ucmd->result)) 358 return -EFAULT; 359 } 360 361 return status; 362 } 363 364 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 365 struct nvme_passthru_cmd64 __user *ucmd, unsigned int flags, 366 bool open_for_write) 367 { 368 struct nvme_passthru_cmd64 cmd; 369 struct nvme_command c; 370 unsigned timeout = 0; 371 int status; 372 373 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 374 return -EFAULT; 375 if (cmd.flags) 376 return -EINVAL; 377 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid)) 378 return -EINVAL; 379 380 memset(&c, 0, sizeof(c)); 381 c.common.opcode = cmd.opcode; 382 c.common.flags = cmd.flags; 383 c.common.nsid = cpu_to_le32(cmd.nsid); 384 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 385 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 386 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 387 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 388 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 389 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 390 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 391 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 392 393 if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write)) 394 return -EACCES; 395 396 if (cmd.timeout_ms) 397 timeout = msecs_to_jiffies(cmd.timeout_ms); 398 399 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 400 cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata), 401 cmd.metadata_len, &cmd.result, timeout, flags); 402 403 if (status >= 0) { 404 if (put_user(cmd.result, &ucmd->result)) 405 return -EFAULT; 406 } 407 408 return status; 409 } 410 411 struct nvme_uring_data { 412 __u64 metadata; 413 __u64 addr; 414 __u32 data_len; 415 __u32 metadata_len; 416 __u32 timeout_ms; 417 }; 418 419 /* 420 * This overlays struct io_uring_cmd pdu. 421 * Expect build errors if this grows larger than that. 422 */ 423 struct nvme_uring_cmd_pdu { 424 struct request *req; 425 struct bio *bio; 426 u64 result; 427 int status; 428 }; 429 430 static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu( 431 struct io_uring_cmd *ioucmd) 432 { 433 return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu); 434 } 435 436 static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw) 437 { 438 struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req); 439 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 440 441 if (pdu->bio) 442 blk_rq_unmap_user(pdu->bio); 443 io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 444 IO_URING_CMD_TASK_WORK_ISSUE_FLAGS); 445 } 446 447 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req, 448 blk_status_t err, 449 const struct io_comp_batch *iob) 450 { 451 struct io_uring_cmd *ioucmd = req->end_io_data; 452 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 453 454 if (nvme_req(req)->flags & NVME_REQ_CANCELLED) { 455 pdu->status = -EINTR; 456 } else { 457 pdu->status = nvme_req(req)->status; 458 if (!pdu->status) 459 pdu->status = blk_status_to_errno(err); 460 } 461 pdu->result = le64_to_cpu(nvme_req(req)->result.u64); 462 463 /* 464 * For IOPOLL, check if this completion is happening in the context 465 * of the same io_ring that owns the request (local context). If so, 466 * we can complete inline without task_work overhead. Otherwise, we 467 * must punt to task_work to ensure completion happens in the correct 468 * ring's context. 469 */ 470 if (blk_rq_is_poll(req) && iob && 471 iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) { 472 if (pdu->bio) 473 blk_rq_unmap_user(pdu->bio); 474 io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0); 475 } else { 476 io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb); 477 } 478 return RQ_END_IO_FREE; 479 } 480 481 static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 482 struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec) 483 { 484 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 485 const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, 486 struct nvme_uring_cmd); 487 struct request_queue *q = ns ? ns->queue : ctrl->admin_q; 488 bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; 489 struct nvme_uring_data d; 490 struct nvme_command c; 491 struct iov_iter iter; 492 struct iov_iter *map_iter = NULL; 493 struct request *req; 494 blk_opf_t rq_flags = 0; 495 blk_mq_req_flags_t blk_flags = 0; 496 int ret; 497 498 c.common.opcode = READ_ONCE(cmd->opcode); 499 c.common.flags = READ_ONCE(cmd->flags); 500 if (c.common.flags) 501 return -EINVAL; 502 503 c.common.command_id = 0; 504 c.common.nsid = cpu_to_le32(cmd->nsid); 505 if (!nvme_validate_passthru_nsid(ctrl, ns, le32_to_cpu(c.common.nsid))) 506 return -EINVAL; 507 508 c.common.cdw2[0] = cpu_to_le32(READ_ONCE(cmd->cdw2)); 509 c.common.cdw2[1] = cpu_to_le32(READ_ONCE(cmd->cdw3)); 510 c.common.metadata = 0; 511 c.common.dptr.prp1 = c.common.dptr.prp2 = 0; 512 c.common.cdw10 = cpu_to_le32(READ_ONCE(cmd->cdw10)); 513 c.common.cdw11 = cpu_to_le32(READ_ONCE(cmd->cdw11)); 514 c.common.cdw12 = cpu_to_le32(READ_ONCE(cmd->cdw12)); 515 c.common.cdw13 = cpu_to_le32(READ_ONCE(cmd->cdw13)); 516 c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); 517 c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); 518 519 if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) 520 return -EACCES; 521 522 d.metadata = READ_ONCE(cmd->metadata); 523 d.addr = READ_ONCE(cmd->addr); 524 d.data_len = READ_ONCE(cmd->data_len); 525 d.metadata_len = READ_ONCE(cmd->metadata_len); 526 d.timeout_ms = READ_ONCE(cmd->timeout_ms); 527 528 if (d.data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) { 529 int ddir = nvme_is_write(&c) ? WRITE : READ; 530 531 if (vec) 532 ret = io_uring_cmd_import_fixed_vec(ioucmd, 533 u64_to_user_ptr(d.addr), d.data_len, 534 ddir, &iter, issue_flags); 535 else 536 ret = io_uring_cmd_import_fixed(d.addr, d.data_len, 537 ddir, &iter, ioucmd, issue_flags); 538 if (ret < 0) 539 return ret; 540 541 map_iter = &iter; 542 } 543 544 if (issue_flags & IO_URING_F_NONBLOCK) { 545 rq_flags |= REQ_NOWAIT; 546 blk_flags = BLK_MQ_REQ_NOWAIT; 547 } 548 if (issue_flags & IO_URING_F_IOPOLL) 549 rq_flags |= REQ_POLLED; 550 551 req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags); 552 if (IS_ERR(req)) 553 return PTR_ERR(req); 554 req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0; 555 556 if (d.data_len) { 557 ret = nvme_map_user_request(req, d.addr, d.data_len, 558 nvme_to_user_ptr(d.metadata), d.metadata_len, 559 map_iter, vec ? NVME_IOCTL_VEC : 0); 560 if (ret) 561 goto out_free_req; 562 } 563 564 /* to free bio on completion, as req->bio will be null at that time */ 565 pdu->bio = req->bio; 566 pdu->req = req; 567 req->end_io_data = ioucmd; 568 req->end_io = nvme_uring_cmd_end_io; 569 blk_execute_rq_nowait(req, false); 570 return -EIOCBQUEUED; 571 572 out_free_req: 573 blk_mq_free_request(req); 574 return ret; 575 } 576 577 static bool is_ctrl_ioctl(unsigned int cmd) 578 { 579 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD) 580 return true; 581 if (is_sed_ioctl(cmd)) 582 return true; 583 return false; 584 } 585 586 static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd, 587 void __user *argp, bool open_for_write) 588 { 589 switch (cmd) { 590 case NVME_IOCTL_ADMIN_CMD: 591 return nvme_user_cmd(ctrl, NULL, argp, 0, open_for_write); 592 case NVME_IOCTL_ADMIN64_CMD: 593 return nvme_user_cmd64(ctrl, NULL, argp, 0, open_for_write); 594 default: 595 return sed_ioctl(ctrl->opal_dev, cmd, argp); 596 } 597 } 598 599 #ifdef COMPAT_FOR_U64_ALIGNMENT 600 struct nvme_user_io32 { 601 __u8 opcode; 602 __u8 flags; 603 __u16 control; 604 __u16 nblocks; 605 __u16 rsvd; 606 __u64 metadata; 607 __u64 addr; 608 __u64 slba; 609 __u32 dsmgmt; 610 __u32 reftag; 611 __u16 apptag; 612 __u16 appmask; 613 } __attribute__((__packed__)); 614 #define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32) 615 #endif /* COMPAT_FOR_U64_ALIGNMENT */ 616 617 static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, 618 void __user *argp, unsigned int flags, bool open_for_write) 619 { 620 switch (cmd) { 621 case NVME_IOCTL_ID: 622 force_successful_syscall_return(); 623 return ns->head->ns_id; 624 case NVME_IOCTL_IO_CMD: 625 return nvme_user_cmd(ns->ctrl, ns, argp, flags, open_for_write); 626 /* 627 * struct nvme_user_io can have different padding on some 32-bit ABIs. 628 * Just accept the compat version as all fields that are used are the 629 * same size and at the same offset. 630 */ 631 #ifdef COMPAT_FOR_U64_ALIGNMENT 632 case NVME_IOCTL_SUBMIT_IO32: 633 #endif 634 case NVME_IOCTL_SUBMIT_IO: 635 return nvme_submit_io(ns, argp, flags, open_for_write); 636 case NVME_IOCTL_IO64_CMD_VEC: 637 flags |= NVME_IOCTL_VEC; 638 fallthrough; 639 case NVME_IOCTL_IO64_CMD: 640 return nvme_user_cmd64(ns->ctrl, ns, argp, flags, 641 open_for_write); 642 default: 643 return -ENOTTY; 644 } 645 } 646 647 int nvme_ioctl(struct block_device *bdev, blk_mode_t mode, 648 unsigned int cmd, unsigned long arg) 649 { 650 struct nvme_ns *ns = bdev->bd_disk->private_data; 651 bool open_for_write = mode & BLK_OPEN_WRITE; 652 void __user *argp = (void __user *)arg; 653 unsigned int flags = 0; 654 655 if (bdev_is_partition(bdev)) 656 flags |= NVME_IOCTL_PARTITION; 657 658 if (is_ctrl_ioctl(cmd)) 659 return nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write); 660 return nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write); 661 } 662 663 long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 664 { 665 struct nvme_ns *ns = 666 container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev); 667 bool open_for_write = file->f_mode & FMODE_WRITE; 668 void __user *argp = (void __user *)arg; 669 670 if (is_ctrl_ioctl(cmd)) 671 return nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write); 672 return nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write); 673 } 674 675 static int nvme_uring_cmd_checks(unsigned int issue_flags) 676 { 677 678 /* NVMe passthrough requires big SQE/CQE support */ 679 if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) != 680 (IO_URING_F_SQE128|IO_URING_F_CQE32)) 681 return -EOPNOTSUPP; 682 return 0; 683 } 684 685 static int nvme_ns_uring_cmd(struct nvme_ns *ns, struct io_uring_cmd *ioucmd, 686 unsigned int issue_flags) 687 { 688 struct nvme_ctrl *ctrl = ns->ctrl; 689 int ret; 690 691 ret = nvme_uring_cmd_checks(issue_flags); 692 if (ret) 693 return ret; 694 695 switch (ioucmd->cmd_op) { 696 case NVME_URING_CMD_IO: 697 ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, false); 698 break; 699 case NVME_URING_CMD_IO_VEC: 700 ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, true); 701 break; 702 default: 703 ret = -ENOTTY; 704 } 705 706 return ret; 707 } 708 709 int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) 710 { 711 struct nvme_ns *ns = container_of(file_inode(ioucmd->file)->i_cdev, 712 struct nvme_ns, cdev); 713 714 return nvme_ns_uring_cmd(ns, ioucmd, issue_flags); 715 } 716 717 int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, 718 struct io_comp_batch *iob, 719 unsigned int poll_flags) 720 { 721 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); 722 struct request *req = pdu->req; 723 724 if (req && blk_rq_is_poll(req)) 725 return blk_rq_poll(req, iob, poll_flags); 726 return 0; 727 } 728 #ifdef CONFIG_NVME_MULTIPATH 729 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, 730 void __user *argp, struct nvme_ns_head *head, int srcu_idx, 731 bool open_for_write) 732 __releases_shared(&head->srcu) 733 { 734 struct nvme_ctrl *ctrl = ns->ctrl; 735 int ret; 736 737 nvme_get_ctrl(ns->ctrl); 738 srcu_read_unlock(&head->srcu, srcu_idx); 739 ret = nvme_ctrl_ioctl(ctrl, cmd, argp, open_for_write); 740 741 nvme_put_ctrl(ctrl); 742 return ret; 743 } 744 745 int nvme_ns_head_ioctl(struct block_device *bdev, blk_mode_t mode, 746 unsigned int cmd, unsigned long arg) 747 { 748 struct nvme_ns_head *head = bdev->bd_disk->private_data; 749 bool open_for_write = mode & BLK_OPEN_WRITE; 750 void __user *argp = (void __user *)arg; 751 struct nvme_ns *ns; 752 int srcu_idx, ret = -EWOULDBLOCK; 753 unsigned int flags = 0; 754 755 if (bdev_is_partition(bdev)) 756 flags |= NVME_IOCTL_PARTITION; 757 758 srcu_idx = srcu_read_lock(&head->srcu); 759 ns = nvme_find_path(head); 760 if (!ns) 761 goto out_unlock; 762 763 /* 764 * Handle ioctls that apply to the controller instead of the namespace 765 * separately and drop the ns SRCU reference early. This avoids a 766 * deadlock when deleting namespaces using the passthrough interface. 767 */ 768 if (is_ctrl_ioctl(cmd)) 769 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx, 770 open_for_write); 771 772 ret = nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write); 773 out_unlock: 774 srcu_read_unlock(&head->srcu, srcu_idx); 775 return ret; 776 } 777 778 long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, 779 unsigned long arg) 780 { 781 bool open_for_write = file->f_mode & FMODE_WRITE; 782 struct cdev *cdev = file_inode(file)->i_cdev; 783 struct nvme_ns_head *head = 784 container_of(cdev, struct nvme_ns_head, cdev); 785 void __user *argp = (void __user *)arg; 786 struct nvme_ns *ns; 787 int srcu_idx, ret = -EWOULDBLOCK; 788 789 srcu_idx = srcu_read_lock(&head->srcu); 790 ns = nvme_find_path(head); 791 if (!ns) 792 goto out_unlock; 793 794 if (is_ctrl_ioctl(cmd)) 795 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx, 796 open_for_write); 797 798 ret = nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write); 799 out_unlock: 800 srcu_read_unlock(&head->srcu, srcu_idx); 801 return ret; 802 } 803 804 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, 805 unsigned int issue_flags) 806 { 807 struct cdev *cdev = file_inode(ioucmd->file)->i_cdev; 808 struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev); 809 int srcu_idx = srcu_read_lock(&head->srcu); 810 struct nvme_ns *ns = nvme_find_path(head); 811 int ret = -EINVAL; 812 813 if (ns) 814 ret = nvme_ns_uring_cmd(ns, ioucmd, issue_flags); 815 srcu_read_unlock(&head->srcu, srcu_idx); 816 return ret; 817 } 818 #endif /* CONFIG_NVME_MULTIPATH */ 819 820 int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) 821 { 822 struct nvme_ctrl *ctrl = ioucmd->file->private_data; 823 int ret; 824 825 ret = nvme_uring_cmd_checks(issue_flags); 826 if (ret) 827 return ret; 828 829 switch (ioucmd->cmd_op) { 830 case NVME_URING_CMD_ADMIN: 831 ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, false); 832 break; 833 case NVME_URING_CMD_ADMIN_VEC: 834 ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, true); 835 break; 836 default: 837 ret = -ENOTTY; 838 } 839 840 return ret; 841 } 842 843 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp, 844 bool open_for_write) 845 { 846 struct nvme_ns *ns; 847 int ret, srcu_idx; 848 849 srcu_idx = srcu_read_lock(&ctrl->srcu); 850 if (list_empty(&ctrl->namespaces)) { 851 ret = -ENOTTY; 852 goto out_unlock; 853 } 854 855 ns = list_first_or_null_rcu(&ctrl->namespaces, struct nvme_ns, list); 856 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) { 857 dev_warn(ctrl->device, 858 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n"); 859 ret = -EINVAL; 860 goto out_unlock; 861 } 862 863 dev_warn(ctrl->device, 864 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n"); 865 if (!nvme_get_ns(ns)) { 866 ret = -ENXIO; 867 goto out_unlock; 868 } 869 srcu_read_unlock(&ctrl->srcu, srcu_idx); 870 871 ret = nvme_user_cmd(ctrl, ns, argp, 0, open_for_write); 872 nvme_put_ns(ns); 873 return ret; 874 875 out_unlock: 876 srcu_read_unlock(&ctrl->srcu, srcu_idx); 877 return ret; 878 } 879 880 long nvme_dev_ioctl(struct file *file, unsigned int cmd, 881 unsigned long arg) 882 { 883 bool open_for_write = file->f_mode & FMODE_WRITE; 884 struct nvme_ctrl *ctrl = file->private_data; 885 void __user *argp = (void __user *)arg; 886 887 switch (cmd) { 888 case NVME_IOCTL_ADMIN_CMD: 889 return nvme_user_cmd(ctrl, NULL, argp, 0, open_for_write); 890 case NVME_IOCTL_ADMIN64_CMD: 891 return nvme_user_cmd64(ctrl, NULL, argp, 0, open_for_write); 892 case NVME_IOCTL_IO_CMD: 893 return nvme_dev_user_cmd(ctrl, argp, open_for_write); 894 case NVME_IOCTL_RESET: 895 if (!capable(CAP_SYS_ADMIN)) 896 return -EACCES; 897 dev_warn(ctrl->device, "resetting controller\n"); 898 return nvme_reset_ctrl_sync(ctrl); 899 case NVME_IOCTL_SUBSYS_RESET: 900 if (!capable(CAP_SYS_ADMIN)) 901 return -EACCES; 902 return nvme_reset_subsystem(ctrl); 903 case NVME_IOCTL_RESCAN: 904 if (!capable(CAP_SYS_ADMIN)) 905 return -EACCES; 906 nvme_queue_scan(ctrl); 907 return 0; 908 default: 909 return -ENOTTY; 910 } 911 } 912