1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Description: uring_cmd based ublk 4 */ 5 6 #include "kublk.h" 7 8 #define MAX_NR_TGT_ARG 64 9 10 unsigned int ublk_dbg_mask = UBLK_LOG; 11 static const struct ublk_tgt_ops *tgt_ops_list[] = { 12 &null_tgt_ops, 13 &loop_tgt_ops, 14 &stripe_tgt_ops, 15 &fault_inject_tgt_ops, 16 }; 17 18 static const struct ublk_tgt_ops *ublk_find_tgt(const char *name) 19 { 20 int i; 21 22 if (name == NULL) 23 return NULL; 24 25 for (i = 0; i < ARRAY_SIZE(tgt_ops_list); i++) 26 if (strcmp(tgt_ops_list[i]->name, name) == 0) 27 return tgt_ops_list[i]; 28 return NULL; 29 } 30 31 static inline int ublk_setup_ring(struct io_uring *r, int depth, 32 int cq_depth, unsigned flags) 33 { 34 struct io_uring_params p; 35 36 memset(&p, 0, sizeof(p)); 37 p.flags = flags | IORING_SETUP_CQSIZE; 38 p.cq_entries = cq_depth; 39 40 return io_uring_queue_init_params(depth, r, &p); 41 } 42 43 static void ublk_ctrl_init_cmd(struct ublk_dev *dev, 44 struct io_uring_sqe *sqe, 45 struct ublk_ctrl_cmd_data *data) 46 { 47 struct ublksrv_ctrl_dev_info *info = &dev->dev_info; 48 struct ublksrv_ctrl_cmd *cmd = (struct ublksrv_ctrl_cmd *)ublk_get_sqe_cmd(sqe); 49 50 sqe->fd = dev->ctrl_fd; 51 sqe->opcode = IORING_OP_URING_CMD; 52 sqe->ioprio = 0; 53 54 if (data->flags & CTRL_CMD_HAS_BUF) { 55 cmd->addr = data->addr; 56 cmd->len = data->len; 57 } 58 59 if (data->flags & CTRL_CMD_HAS_DATA) 60 cmd->data[0] = data->data[0]; 61 62 cmd->dev_id = info->dev_id; 63 cmd->queue_id = -1; 64 65 ublk_set_sqe_cmd_op(sqe, data->cmd_op); 66 67 io_uring_sqe_set_data(sqe, cmd); 68 } 69 70 static int __ublk_ctrl_cmd(struct ublk_dev *dev, 71 struct ublk_ctrl_cmd_data *data) 72 { 73 struct io_uring_sqe *sqe; 74 struct io_uring_cqe *cqe; 75 int ret = -EINVAL; 76 77 sqe = io_uring_get_sqe(&dev->ring); 78 if (!sqe) { 79 ublk_err("%s: can't get sqe ret %d\n", __func__, ret); 80 return ret; 81 } 82 83 ublk_ctrl_init_cmd(dev, sqe, data); 84 85 ret = io_uring_submit(&dev->ring); 86 if (ret < 0) { 87 ublk_err("uring submit ret %d\n", ret); 88 return ret; 89 } 90 91 ret = io_uring_wait_cqe(&dev->ring, &cqe); 92 if (ret < 0) { 93 ublk_err("wait cqe: %s\n", strerror(-ret)); 94 return ret; 95 } 96 io_uring_cqe_seen(&dev->ring, cqe); 97 98 return cqe->res; 99 } 100 101 static int ublk_ctrl_stop_dev(struct ublk_dev *dev) 102 { 103 struct ublk_ctrl_cmd_data data = { 104 .cmd_op = UBLK_U_CMD_STOP_DEV, 105 }; 106 107 return __ublk_ctrl_cmd(dev, &data); 108 } 109 110 static int ublk_ctrl_start_dev(struct ublk_dev *dev, 111 int daemon_pid) 112 { 113 struct ublk_ctrl_cmd_data data = { 114 .cmd_op = UBLK_U_CMD_START_DEV, 115 .flags = CTRL_CMD_HAS_DATA, 116 }; 117 118 dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid; 119 120 return __ublk_ctrl_cmd(dev, &data); 121 } 122 123 static int ublk_ctrl_start_user_recovery(struct ublk_dev *dev) 124 { 125 struct ublk_ctrl_cmd_data data = { 126 .cmd_op = UBLK_U_CMD_START_USER_RECOVERY, 127 }; 128 129 return __ublk_ctrl_cmd(dev, &data); 130 } 131 132 static int ublk_ctrl_end_user_recovery(struct ublk_dev *dev, int daemon_pid) 133 { 134 struct ublk_ctrl_cmd_data data = { 135 .cmd_op = UBLK_U_CMD_END_USER_RECOVERY, 136 .flags = CTRL_CMD_HAS_DATA, 137 }; 138 139 dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid; 140 141 return __ublk_ctrl_cmd(dev, &data); 142 } 143 144 static int ublk_ctrl_add_dev(struct ublk_dev *dev) 145 { 146 struct ublk_ctrl_cmd_data data = { 147 .cmd_op = UBLK_U_CMD_ADD_DEV, 148 .flags = CTRL_CMD_HAS_BUF, 149 .addr = (__u64) (uintptr_t) &dev->dev_info, 150 .len = sizeof(struct ublksrv_ctrl_dev_info), 151 }; 152 153 return __ublk_ctrl_cmd(dev, &data); 154 } 155 156 static int ublk_ctrl_del_dev(struct ublk_dev *dev) 157 { 158 struct ublk_ctrl_cmd_data data = { 159 .cmd_op = UBLK_U_CMD_DEL_DEV, 160 .flags = 0, 161 }; 162 163 return __ublk_ctrl_cmd(dev, &data); 164 } 165 166 static int ublk_ctrl_get_info(struct ublk_dev *dev) 167 { 168 struct ublk_ctrl_cmd_data data = { 169 .cmd_op = UBLK_U_CMD_GET_DEV_INFO, 170 .flags = CTRL_CMD_HAS_BUF, 171 .addr = (__u64) (uintptr_t) &dev->dev_info, 172 .len = sizeof(struct ublksrv_ctrl_dev_info), 173 }; 174 175 return __ublk_ctrl_cmd(dev, &data); 176 } 177 178 static int ublk_ctrl_set_params(struct ublk_dev *dev, 179 struct ublk_params *params) 180 { 181 struct ublk_ctrl_cmd_data data = { 182 .cmd_op = UBLK_U_CMD_SET_PARAMS, 183 .flags = CTRL_CMD_HAS_BUF, 184 .addr = (__u64) (uintptr_t) params, 185 .len = sizeof(*params), 186 }; 187 params->len = sizeof(*params); 188 return __ublk_ctrl_cmd(dev, &data); 189 } 190 191 static int ublk_ctrl_get_params(struct ublk_dev *dev, 192 struct ublk_params *params) 193 { 194 struct ublk_ctrl_cmd_data data = { 195 .cmd_op = UBLK_U_CMD_GET_PARAMS, 196 .flags = CTRL_CMD_HAS_BUF, 197 .addr = (__u64)params, 198 .len = sizeof(*params), 199 }; 200 201 params->len = sizeof(*params); 202 203 return __ublk_ctrl_cmd(dev, &data); 204 } 205 206 static int ublk_ctrl_get_features(struct ublk_dev *dev, 207 __u64 *features) 208 { 209 struct ublk_ctrl_cmd_data data = { 210 .cmd_op = UBLK_U_CMD_GET_FEATURES, 211 .flags = CTRL_CMD_HAS_BUF, 212 .addr = (__u64) (uintptr_t) features, 213 .len = sizeof(*features), 214 }; 215 216 return __ublk_ctrl_cmd(dev, &data); 217 } 218 219 static int ublk_ctrl_update_size(struct ublk_dev *dev, 220 __u64 nr_sects) 221 { 222 struct ublk_ctrl_cmd_data data = { 223 .cmd_op = UBLK_U_CMD_UPDATE_SIZE, 224 .flags = CTRL_CMD_HAS_DATA, 225 }; 226 227 data.data[0] = nr_sects; 228 return __ublk_ctrl_cmd(dev, &data); 229 } 230 231 static int ublk_ctrl_quiesce_dev(struct ublk_dev *dev, 232 unsigned int timeout_ms) 233 { 234 struct ublk_ctrl_cmd_data data = { 235 .cmd_op = UBLK_U_CMD_QUIESCE_DEV, 236 .flags = CTRL_CMD_HAS_DATA, 237 }; 238 239 data.data[0] = timeout_ms; 240 return __ublk_ctrl_cmd(dev, &data); 241 } 242 243 static const char *ublk_dev_state_desc(struct ublk_dev *dev) 244 { 245 switch (dev->dev_info.state) { 246 case UBLK_S_DEV_DEAD: 247 return "DEAD"; 248 case UBLK_S_DEV_LIVE: 249 return "LIVE"; 250 case UBLK_S_DEV_QUIESCED: 251 return "QUIESCED"; 252 default: 253 return "UNKNOWN"; 254 }; 255 } 256 257 static void ublk_print_cpu_set(const cpu_set_t *set, char *buf, unsigned len) 258 { 259 unsigned done = 0; 260 int i; 261 262 for (i = 0; i < CPU_SETSIZE; i++) { 263 if (CPU_ISSET(i, set)) 264 done += snprintf(&buf[done], len - done, "%d ", i); 265 } 266 } 267 268 static void ublk_adjust_affinity(cpu_set_t *set) 269 { 270 int j, updated = 0; 271 272 /* 273 * Just keep the 1st CPU now. 274 * 275 * In future, auto affinity selection can be tried. 276 */ 277 for (j = 0; j < CPU_SETSIZE; j++) { 278 if (CPU_ISSET(j, set)) { 279 if (!updated) { 280 updated = 1; 281 continue; 282 } 283 CPU_CLR(j, set); 284 } 285 } 286 } 287 288 /* Caller must free the allocated buffer */ 289 static int ublk_ctrl_get_affinity(struct ublk_dev *ctrl_dev, cpu_set_t **ptr_buf) 290 { 291 struct ublk_ctrl_cmd_data data = { 292 .cmd_op = UBLK_U_CMD_GET_QUEUE_AFFINITY, 293 .flags = CTRL_CMD_HAS_DATA | CTRL_CMD_HAS_BUF, 294 }; 295 cpu_set_t *buf; 296 int i, ret; 297 298 buf = malloc(sizeof(cpu_set_t) * ctrl_dev->dev_info.nr_hw_queues); 299 if (!buf) 300 return -ENOMEM; 301 302 for (i = 0; i < ctrl_dev->dev_info.nr_hw_queues; i++) { 303 data.data[0] = i; 304 data.len = sizeof(cpu_set_t); 305 data.addr = (__u64)&buf[i]; 306 307 ret = __ublk_ctrl_cmd(ctrl_dev, &data); 308 if (ret < 0) { 309 free(buf); 310 return ret; 311 } 312 ublk_adjust_affinity(&buf[i]); 313 } 314 315 *ptr_buf = buf; 316 return 0; 317 } 318 319 static void ublk_ctrl_dump(struct ublk_dev *dev) 320 { 321 struct ublksrv_ctrl_dev_info *info = &dev->dev_info; 322 struct ublk_params p; 323 cpu_set_t *affinity; 324 int ret; 325 326 ret = ublk_ctrl_get_params(dev, &p); 327 if (ret < 0) { 328 ublk_err("failed to get params %d %s\n", ret, strerror(-ret)); 329 return; 330 } 331 332 ret = ublk_ctrl_get_affinity(dev, &affinity); 333 if (ret < 0) { 334 ublk_err("failed to get affinity %m\n"); 335 return; 336 } 337 338 ublk_log("dev id %d: nr_hw_queues %d queue_depth %d block size %d dev_capacity %lld\n", 339 info->dev_id, info->nr_hw_queues, info->queue_depth, 340 1 << p.basic.logical_bs_shift, p.basic.dev_sectors); 341 ublk_log("\tmax rq size %d daemon pid %d flags 0x%llx state %s\n", 342 info->max_io_buf_bytes, info->ublksrv_pid, info->flags, 343 ublk_dev_state_desc(dev)); 344 345 if (affinity) { 346 char buf[512]; 347 int i; 348 349 for (i = 0; i < info->nr_hw_queues; i++) { 350 ublk_print_cpu_set(&affinity[i], buf, sizeof(buf)); 351 printf("\tqueue %u: affinity(%s)\n", 352 i, buf); 353 } 354 free(affinity); 355 } 356 357 fflush(stdout); 358 } 359 360 static void ublk_ctrl_deinit(struct ublk_dev *dev) 361 { 362 close(dev->ctrl_fd); 363 free(dev); 364 } 365 366 static struct ublk_dev *ublk_ctrl_init(void) 367 { 368 struct ublk_dev *dev = (struct ublk_dev *)calloc(1, sizeof(*dev)); 369 struct ublksrv_ctrl_dev_info *info = &dev->dev_info; 370 int ret; 371 372 dev->ctrl_fd = open(CTRL_DEV, O_RDWR); 373 if (dev->ctrl_fd < 0) { 374 free(dev); 375 return NULL; 376 } 377 378 info->max_io_buf_bytes = UBLK_IO_MAX_BYTES; 379 380 ret = ublk_setup_ring(&dev->ring, UBLK_CTRL_RING_DEPTH, 381 UBLK_CTRL_RING_DEPTH, IORING_SETUP_SQE128); 382 if (ret < 0) { 383 ublk_err("queue_init: %s\n", strerror(-ret)); 384 free(dev); 385 return NULL; 386 } 387 dev->nr_fds = 1; 388 389 return dev; 390 } 391 392 static int __ublk_queue_cmd_buf_sz(unsigned depth) 393 { 394 int size = depth * sizeof(struct ublksrv_io_desc); 395 unsigned int page_sz = getpagesize(); 396 397 return round_up(size, page_sz); 398 } 399 400 static int ublk_queue_max_cmd_buf_sz(void) 401 { 402 return __ublk_queue_cmd_buf_sz(UBLK_MAX_QUEUE_DEPTH); 403 } 404 405 static int ublk_queue_cmd_buf_sz(struct ublk_queue *q) 406 { 407 return __ublk_queue_cmd_buf_sz(q->q_depth); 408 } 409 410 static void ublk_queue_deinit(struct ublk_queue *q) 411 { 412 int i; 413 int nr_ios = q->q_depth; 414 415 if (q->io_cmd_buf) 416 munmap(q->io_cmd_buf, ublk_queue_cmd_buf_sz(q)); 417 418 for (i = 0; i < nr_ios; i++) 419 free(q->ios[i].buf_addr); 420 } 421 422 static void ublk_thread_deinit(struct ublk_thread *t) 423 { 424 io_uring_unregister_buffers(&t->ring); 425 426 io_uring_unregister_ring_fd(&t->ring); 427 428 if (t->ring.ring_fd > 0) { 429 io_uring_unregister_files(&t->ring); 430 close(t->ring.ring_fd); 431 t->ring.ring_fd = -1; 432 } 433 } 434 435 static int ublk_queue_init(struct ublk_queue *q, unsigned long long extra_flags) 436 { 437 struct ublk_dev *dev = q->dev; 438 int depth = dev->dev_info.queue_depth; 439 int i; 440 int cmd_buf_size, io_buf_size; 441 unsigned long off; 442 443 q->tgt_ops = dev->tgt.ops; 444 q->flags = 0; 445 q->q_depth = depth; 446 q->flags = dev->dev_info.flags; 447 q->flags |= extra_flags; 448 449 /* Cache fd in queue for fast path access */ 450 q->ublk_fd = dev->fds[0]; 451 452 cmd_buf_size = ublk_queue_cmd_buf_sz(q); 453 off = UBLKSRV_CMD_BUF_OFFSET + q->q_id * ublk_queue_max_cmd_buf_sz(); 454 q->io_cmd_buf = mmap(0, cmd_buf_size, PROT_READ, 455 MAP_SHARED | MAP_POPULATE, dev->fds[0], off); 456 if (q->io_cmd_buf == MAP_FAILED) { 457 ublk_err("ublk dev %d queue %d map io_cmd_buf failed %m\n", 458 q->dev->dev_info.dev_id, q->q_id); 459 goto fail; 460 } 461 462 io_buf_size = dev->dev_info.max_io_buf_bytes; 463 for (i = 0; i < q->q_depth; i++) { 464 q->ios[i].buf_addr = NULL; 465 q->ios[i].flags = UBLKS_IO_NEED_FETCH_RQ | UBLKS_IO_FREE; 466 q->ios[i].tag = i; 467 468 if (ublk_queue_no_buf(q)) 469 continue; 470 471 if (posix_memalign((void **)&q->ios[i].buf_addr, 472 getpagesize(), io_buf_size)) { 473 ublk_err("ublk dev %d queue %d io %d posix_memalign failed %m\n", 474 dev->dev_info.dev_id, q->q_id, i); 475 goto fail; 476 } 477 } 478 479 return 0; 480 fail: 481 ublk_queue_deinit(q); 482 ublk_err("ublk dev %d queue %d failed\n", 483 dev->dev_info.dev_id, q->q_id); 484 return -ENOMEM; 485 } 486 487 static int ublk_thread_init(struct ublk_thread *t, unsigned long long extra_flags) 488 { 489 struct ublk_dev *dev = t->dev; 490 unsigned long long flags = dev->dev_info.flags | extra_flags; 491 int ring_depth = dev->tgt.sq_depth, cq_depth = dev->tgt.cq_depth; 492 int ret; 493 494 ret = ublk_setup_ring(&t->ring, ring_depth, cq_depth, 495 IORING_SETUP_COOP_TASKRUN | 496 IORING_SETUP_SINGLE_ISSUER | 497 IORING_SETUP_DEFER_TASKRUN); 498 if (ret < 0) { 499 ublk_err("ublk dev %d thread %d setup io_uring failed %d\n", 500 dev->dev_info.dev_id, t->idx, ret); 501 goto fail; 502 } 503 504 if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) { 505 unsigned nr_ios = dev->dev_info.queue_depth * dev->dev_info.nr_hw_queues; 506 unsigned max_nr_ios_per_thread = nr_ios / dev->nthreads; 507 max_nr_ios_per_thread += !!(nr_ios % dev->nthreads); 508 ret = io_uring_register_buffers_sparse( 509 &t->ring, max_nr_ios_per_thread); 510 if (ret) { 511 ublk_err("ublk dev %d thread %d register spare buffers failed %d", 512 dev->dev_info.dev_id, t->idx, ret); 513 goto fail; 514 } 515 } 516 517 io_uring_register_ring_fd(&t->ring); 518 519 if (flags & UBLKS_Q_NO_UBLK_FIXED_FD) { 520 /* Register only backing files starting from index 1, exclude ublk control device */ 521 if (dev->nr_fds > 1) { 522 ret = io_uring_register_files(&t->ring, &dev->fds[1], dev->nr_fds - 1); 523 } else { 524 /* No backing files to register, skip file registration */ 525 ret = 0; 526 } 527 } else { 528 ret = io_uring_register_files(&t->ring, dev->fds, dev->nr_fds); 529 } 530 if (ret) { 531 ublk_err("ublk dev %d thread %d register files failed %d\n", 532 t->dev->dev_info.dev_id, t->idx, ret); 533 goto fail; 534 } 535 536 return 0; 537 fail: 538 ublk_thread_deinit(t); 539 ublk_err("ublk dev %d thread %d init failed\n", 540 dev->dev_info.dev_id, t->idx); 541 return -ENOMEM; 542 } 543 544 #define WAIT_USEC 100000 545 #define MAX_WAIT_USEC (3 * 1000000) 546 static int ublk_dev_prep(const struct dev_ctx *ctx, struct ublk_dev *dev) 547 { 548 int dev_id = dev->dev_info.dev_id; 549 unsigned int wait_usec = 0; 550 int ret = 0, fd = -1; 551 char buf[64]; 552 553 snprintf(buf, 64, "%s%d", UBLKC_DEV, dev_id); 554 555 while (wait_usec < MAX_WAIT_USEC) { 556 fd = open(buf, O_RDWR); 557 if (fd >= 0) 558 break; 559 usleep(WAIT_USEC); 560 wait_usec += WAIT_USEC; 561 } 562 if (fd < 0) { 563 ublk_err("can't open %s %s\n", buf, strerror(errno)); 564 return -1; 565 } 566 567 dev->fds[0] = fd; 568 if (dev->tgt.ops->init_tgt) 569 ret = dev->tgt.ops->init_tgt(ctx, dev); 570 if (ret) 571 close(dev->fds[0]); 572 return ret; 573 } 574 575 static void ublk_dev_unprep(struct ublk_dev *dev) 576 { 577 if (dev->tgt.ops->deinit_tgt) 578 dev->tgt.ops->deinit_tgt(dev); 579 close(dev->fds[0]); 580 } 581 582 static void ublk_set_auto_buf_reg(const struct ublk_queue *q, 583 struct io_uring_sqe *sqe, 584 unsigned short tag) 585 { 586 struct ublk_auto_buf_reg buf = {}; 587 588 if (q->tgt_ops->buf_index) 589 buf.index = q->tgt_ops->buf_index(q, tag); 590 else 591 buf.index = q->ios[tag].buf_index; 592 593 if (ublk_queue_auto_zc_fallback(q)) 594 buf.flags = UBLK_AUTO_BUF_REG_FALLBACK; 595 596 sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf); 597 } 598 599 int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io) 600 { 601 struct ublk_queue *q = ublk_io_to_queue(io); 602 struct ublksrv_io_cmd *cmd; 603 struct io_uring_sqe *sqe[1]; 604 unsigned int cmd_op = 0; 605 __u64 user_data; 606 607 /* only freed io can be issued */ 608 if (!(io->flags & UBLKS_IO_FREE)) 609 return 0; 610 611 /* 612 * we issue because we need either fetching or committing or 613 * getting data 614 */ 615 if (!(io->flags & 616 (UBLKS_IO_NEED_FETCH_RQ | UBLKS_IO_NEED_COMMIT_RQ_COMP | UBLKS_IO_NEED_GET_DATA))) 617 return 0; 618 619 if (io->flags & UBLKS_IO_NEED_GET_DATA) 620 cmd_op = UBLK_U_IO_NEED_GET_DATA; 621 else if (io->flags & UBLKS_IO_NEED_COMMIT_RQ_COMP) 622 cmd_op = UBLK_U_IO_COMMIT_AND_FETCH_REQ; 623 else if (io->flags & UBLKS_IO_NEED_FETCH_RQ) 624 cmd_op = UBLK_U_IO_FETCH_REQ; 625 626 if (io_uring_sq_space_left(&t->ring) < 1) 627 io_uring_submit(&t->ring); 628 629 ublk_io_alloc_sqes(t, sqe, 1); 630 if (!sqe[0]) { 631 ublk_err("%s: run out of sqe. thread %u, tag %d\n", 632 __func__, t->idx, io->tag); 633 return -1; 634 } 635 636 cmd = (struct ublksrv_io_cmd *)ublk_get_sqe_cmd(sqe[0]); 637 638 if (cmd_op == UBLK_U_IO_COMMIT_AND_FETCH_REQ) 639 cmd->result = io->result; 640 641 /* These fields should be written once, never change */ 642 ublk_set_sqe_cmd_op(sqe[0], cmd_op); 643 sqe[0]->fd = ublk_get_registered_fd(q, 0); /* dev->fds[0] */ 644 sqe[0]->opcode = IORING_OP_URING_CMD; 645 if (q->flags & UBLKS_Q_NO_UBLK_FIXED_FD) 646 sqe[0]->flags = 0; /* Use raw FD, not fixed file */ 647 else 648 sqe[0]->flags = IOSQE_FIXED_FILE; 649 sqe[0]->rw_flags = 0; 650 cmd->tag = io->tag; 651 cmd->q_id = q->q_id; 652 if (!ublk_queue_no_buf(q)) 653 cmd->addr = (__u64) (uintptr_t) io->buf_addr; 654 else 655 cmd->addr = 0; 656 657 if (ublk_queue_use_auto_zc(q)) 658 ublk_set_auto_buf_reg(q, sqe[0], io->tag); 659 660 user_data = build_user_data(io->tag, _IOC_NR(cmd_op), 0, q->q_id, 0); 661 io_uring_sqe_set_data64(sqe[0], user_data); 662 663 io->flags = 0; 664 665 t->cmd_inflight += 1; 666 667 ublk_dbg(UBLK_DBG_IO_CMD, "%s: (thread %u qid %d tag %u cmd_op %u) iof %x stopping %d\n", 668 __func__, t->idx, q->q_id, io->tag, cmd_op, 669 io->flags, !!(t->state & UBLKS_T_STOPPING)); 670 return 1; 671 } 672 673 static void ublk_submit_fetch_commands(struct ublk_thread *t) 674 { 675 struct ublk_queue *q; 676 struct ublk_io *io; 677 int i = 0, j = 0; 678 679 if (t->dev->per_io_tasks) { 680 /* 681 * Lexicographically order all the (qid,tag) pairs, with 682 * qid taking priority (so (1,0) > (0,1)). Then make 683 * this thread the daemon for every Nth entry in this 684 * list (N is the number of threads), starting at this 685 * thread's index. This ensures that each queue is 686 * handled by as many ublk server threads as possible, 687 * so that load that is concentrated on one or a few 688 * queues can make use of all ublk server threads. 689 */ 690 const struct ublksrv_ctrl_dev_info *dinfo = &t->dev->dev_info; 691 int nr_ios = dinfo->nr_hw_queues * dinfo->queue_depth; 692 for (i = t->idx; i < nr_ios; i += t->dev->nthreads) { 693 int q_id = i / dinfo->queue_depth; 694 int tag = i % dinfo->queue_depth; 695 q = &t->dev->q[q_id]; 696 io = &q->ios[tag]; 697 io->buf_index = j++; 698 ublk_queue_io_cmd(t, io); 699 } 700 } else { 701 /* 702 * Service exclusively the queue whose q_id matches our 703 * thread index. 704 */ 705 struct ublk_queue *q = &t->dev->q[t->idx]; 706 for (i = 0; i < q->q_depth; i++) { 707 io = &q->ios[i]; 708 io->buf_index = i; 709 ublk_queue_io_cmd(t, io); 710 } 711 } 712 } 713 714 static int ublk_thread_is_idle(struct ublk_thread *t) 715 { 716 return !io_uring_sq_ready(&t->ring) && !t->io_inflight; 717 } 718 719 static int ublk_thread_is_done(struct ublk_thread *t) 720 { 721 return (t->state & UBLKS_T_STOPPING) && ublk_thread_is_idle(t); 722 } 723 724 static inline void ublksrv_handle_tgt_cqe(struct ublk_thread *t, 725 struct ublk_queue *q, 726 struct io_uring_cqe *cqe) 727 { 728 if (cqe->res < 0 && cqe->res != -EAGAIN) 729 ublk_err("%s: failed tgt io: res %d qid %u tag %u, cmd_op %u\n", 730 __func__, cqe->res, q->q_id, 731 user_data_to_tag(cqe->user_data), 732 user_data_to_op(cqe->user_data)); 733 734 if (q->tgt_ops->tgt_io_done) 735 q->tgt_ops->tgt_io_done(t, q, cqe); 736 } 737 738 static void ublk_handle_uring_cmd(struct ublk_thread *t, 739 struct ublk_queue *q, 740 const struct io_uring_cqe *cqe) 741 { 742 int fetch = (cqe->res != UBLK_IO_RES_ABORT) && 743 !(t->state & UBLKS_T_STOPPING); 744 unsigned tag = user_data_to_tag(cqe->user_data); 745 struct ublk_io *io = &q->ios[tag]; 746 747 if (!fetch) { 748 t->state |= UBLKS_T_STOPPING; 749 io->flags &= ~UBLKS_IO_NEED_FETCH_RQ; 750 } 751 752 if (cqe->res == UBLK_IO_RES_OK) { 753 assert(tag < q->q_depth); 754 if (q->tgt_ops->queue_io) 755 q->tgt_ops->queue_io(t, q, tag); 756 } else if (cqe->res == UBLK_IO_RES_NEED_GET_DATA) { 757 io->flags |= UBLKS_IO_NEED_GET_DATA | UBLKS_IO_FREE; 758 ublk_queue_io_cmd(t, io); 759 } else { 760 /* 761 * COMMIT_REQ will be completed immediately since no fetching 762 * piggyback is required. 763 * 764 * Marking IO_FREE only, then this io won't be issued since 765 * we only issue io with (UBLKS_IO_FREE | UBLKSRV_NEED_*) 766 * 767 * */ 768 io->flags = UBLKS_IO_FREE; 769 } 770 } 771 772 static void ublk_handle_cqe(struct ublk_thread *t, 773 struct io_uring_cqe *cqe, void *data) 774 { 775 struct ublk_dev *dev = t->dev; 776 unsigned q_id = user_data_to_q_id(cqe->user_data); 777 struct ublk_queue *q = &dev->q[q_id]; 778 unsigned cmd_op = user_data_to_op(cqe->user_data); 779 780 if (cqe->res < 0 && cqe->res != -ENODEV) 781 ublk_err("%s: res %d userdata %llx queue state %x\n", __func__, 782 cqe->res, cqe->user_data, q->flags); 783 784 ublk_dbg(UBLK_DBG_IO_CMD, "%s: res %d (qid %d tag %u cmd_op %u target %d/%d) stopping %d\n", 785 __func__, cqe->res, q->q_id, user_data_to_tag(cqe->user_data), 786 cmd_op, is_target_io(cqe->user_data), 787 user_data_to_tgt_data(cqe->user_data), 788 (t->state & UBLKS_T_STOPPING)); 789 790 /* Don't retrieve io in case of target io */ 791 if (is_target_io(cqe->user_data)) { 792 ublksrv_handle_tgt_cqe(t, q, cqe); 793 return; 794 } 795 796 t->cmd_inflight--; 797 798 ublk_handle_uring_cmd(t, q, cqe); 799 } 800 801 static int ublk_reap_events_uring(struct ublk_thread *t) 802 { 803 struct io_uring_cqe *cqe; 804 unsigned head; 805 int count = 0; 806 807 io_uring_for_each_cqe(&t->ring, head, cqe) { 808 ublk_handle_cqe(t, cqe, NULL); 809 count += 1; 810 } 811 io_uring_cq_advance(&t->ring, count); 812 813 return count; 814 } 815 816 static int ublk_process_io(struct ublk_thread *t) 817 { 818 int ret, reapped; 819 820 ublk_dbg(UBLK_DBG_THREAD, "dev%d-t%u: to_submit %d inflight cmd %u stopping %d\n", 821 t->dev->dev_info.dev_id, 822 t->idx, io_uring_sq_ready(&t->ring), 823 t->cmd_inflight, 824 (t->state & UBLKS_T_STOPPING)); 825 826 if (ublk_thread_is_done(t)) 827 return -ENODEV; 828 829 ret = io_uring_submit_and_wait(&t->ring, 1); 830 reapped = ublk_reap_events_uring(t); 831 832 ublk_dbg(UBLK_DBG_THREAD, "submit result %d, reapped %d stop %d idle %d\n", 833 ret, reapped, (t->state & UBLKS_T_STOPPING), 834 (t->state & UBLKS_T_IDLE)); 835 836 return reapped; 837 } 838 839 static void ublk_thread_set_sched_affinity(const struct ublk_thread *t, 840 cpu_set_t *cpuset) 841 { 842 if (sched_setaffinity(0, sizeof(*cpuset), cpuset) < 0) 843 ublk_err("ublk dev %u thread %u set affinity failed", 844 t->dev->dev_info.dev_id, t->idx); 845 } 846 847 struct ublk_thread_info { 848 struct ublk_dev *dev; 849 unsigned idx; 850 sem_t *ready; 851 cpu_set_t *affinity; 852 unsigned long long extra_flags; 853 }; 854 855 static void *ublk_io_handler_fn(void *data) 856 { 857 struct ublk_thread_info *info = data; 858 struct ublk_thread *t = &info->dev->threads[info->idx]; 859 int dev_id = info->dev->dev_info.dev_id; 860 int ret; 861 862 t->dev = info->dev; 863 t->idx = info->idx; 864 865 ret = ublk_thread_init(t, info->extra_flags); 866 if (ret) { 867 ublk_err("ublk dev %d thread %u init failed\n", 868 dev_id, t->idx); 869 return NULL; 870 } 871 /* IO perf is sensitive with queue pthread affinity on NUMA machine*/ 872 if (info->affinity) 873 ublk_thread_set_sched_affinity(t, info->affinity); 874 sem_post(info->ready); 875 876 ublk_dbg(UBLK_DBG_THREAD, "tid %d: ublk dev %d thread %u started\n", 877 gettid(), dev_id, t->idx); 878 879 /* submit all io commands to ublk driver */ 880 ublk_submit_fetch_commands(t); 881 do { 882 if (ublk_process_io(t) < 0) 883 break; 884 } while (1); 885 886 ublk_dbg(UBLK_DBG_THREAD, "tid %d: ublk dev %d thread %d exiting\n", 887 gettid(), dev_id, t->idx); 888 ublk_thread_deinit(t); 889 return NULL; 890 } 891 892 static void ublk_set_parameters(struct ublk_dev *dev) 893 { 894 int ret; 895 896 ret = ublk_ctrl_set_params(dev, &dev->tgt.params); 897 if (ret) 898 ublk_err("dev %d set basic parameter failed %d\n", 899 dev->dev_info.dev_id, ret); 900 } 901 902 static int ublk_send_dev_event(const struct dev_ctx *ctx, struct ublk_dev *dev, int dev_id) 903 { 904 uint64_t id; 905 int evtfd = ctx->_evtfd; 906 907 if (evtfd < 0) 908 return -EBADF; 909 910 if (dev_id >= 0) 911 id = dev_id + 1; 912 else 913 id = ERROR_EVTFD_DEVID; 914 915 if (dev && ctx->shadow_dev) 916 memcpy(&ctx->shadow_dev->q, &dev->q, sizeof(dev->q)); 917 918 if (write(evtfd, &id, sizeof(id)) != sizeof(id)) 919 return -EINVAL; 920 921 close(evtfd); 922 shmdt(ctx->shadow_dev); 923 924 return 0; 925 } 926 927 928 static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) 929 { 930 const struct ublksrv_ctrl_dev_info *dinfo = &dev->dev_info; 931 struct ublk_thread_info *tinfo; 932 unsigned long long extra_flags = 0; 933 cpu_set_t *affinity_buf; 934 void *thread_ret; 935 sem_t ready; 936 int ret, i; 937 938 ublk_dbg(UBLK_DBG_DEV, "%s enter\n", __func__); 939 940 tinfo = calloc(sizeof(struct ublk_thread_info), dev->nthreads); 941 if (!tinfo) 942 return -ENOMEM; 943 944 sem_init(&ready, 0, 0); 945 ret = ublk_dev_prep(ctx, dev); 946 if (ret) 947 return ret; 948 949 ret = ublk_ctrl_get_affinity(dev, &affinity_buf); 950 if (ret) 951 return ret; 952 953 if (ctx->auto_zc_fallback) 954 extra_flags = UBLKS_Q_AUTO_BUF_REG_FALLBACK; 955 if (ctx->no_ublk_fixed_fd) 956 extra_flags |= UBLKS_Q_NO_UBLK_FIXED_FD; 957 958 for (i = 0; i < dinfo->nr_hw_queues; i++) { 959 dev->q[i].dev = dev; 960 dev->q[i].q_id = i; 961 962 ret = ublk_queue_init(&dev->q[i], extra_flags); 963 if (ret) { 964 ublk_err("ublk dev %d queue %d init queue failed\n", 965 dinfo->dev_id, i); 966 goto fail; 967 } 968 } 969 970 for (i = 0; i < dev->nthreads; i++) { 971 tinfo[i].dev = dev; 972 tinfo[i].idx = i; 973 tinfo[i].ready = &ready; 974 tinfo[i].extra_flags = extra_flags; 975 976 /* 977 * If threads are not tied 1:1 to queues, setting thread 978 * affinity based on queue affinity makes little sense. 979 * However, thread CPU affinity has significant impact 980 * on performance, so to compare fairly, we'll still set 981 * thread CPU affinity based on queue affinity where 982 * possible. 983 */ 984 if (dev->nthreads == dinfo->nr_hw_queues) 985 tinfo[i].affinity = &affinity_buf[i]; 986 pthread_create(&dev->threads[i].thread, NULL, 987 ublk_io_handler_fn, 988 &tinfo[i]); 989 } 990 991 for (i = 0; i < dev->nthreads; i++) 992 sem_wait(&ready); 993 free(tinfo); 994 free(affinity_buf); 995 996 /* everything is fine now, start us */ 997 if (ctx->recovery) 998 ret = ublk_ctrl_end_user_recovery(dev, getpid()); 999 else { 1000 ublk_set_parameters(dev); 1001 ret = ublk_ctrl_start_dev(dev, getpid()); 1002 } 1003 if (ret < 0) { 1004 ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret); 1005 goto fail; 1006 } 1007 1008 ublk_ctrl_get_info(dev); 1009 if (ctx->fg) 1010 ublk_ctrl_dump(dev); 1011 else 1012 ublk_send_dev_event(ctx, dev, dev->dev_info.dev_id); 1013 1014 /* wait until we are terminated */ 1015 for (i = 0; i < dev->nthreads; i++) 1016 pthread_join(dev->threads[i].thread, &thread_ret); 1017 fail: 1018 for (i = 0; i < dinfo->nr_hw_queues; i++) 1019 ublk_queue_deinit(&dev->q[i]); 1020 ublk_dev_unprep(dev); 1021 ublk_dbg(UBLK_DBG_DEV, "%s exit\n", __func__); 1022 1023 return ret; 1024 } 1025 1026 static int wait_ublk_dev(const char *path, int evt_mask, unsigned timeout) 1027 { 1028 #define EV_SIZE (sizeof(struct inotify_event)) 1029 #define EV_BUF_LEN (128 * (EV_SIZE + 16)) 1030 struct pollfd pfd; 1031 int fd, wd; 1032 int ret = -EINVAL; 1033 const char *dev_name = basename(path); 1034 1035 fd = inotify_init(); 1036 if (fd < 0) { 1037 ublk_dbg(UBLK_DBG_DEV, "%s: inotify init failed\n", __func__); 1038 return fd; 1039 } 1040 1041 wd = inotify_add_watch(fd, "/dev", evt_mask); 1042 if (wd == -1) { 1043 ublk_dbg(UBLK_DBG_DEV, "%s: add watch for /dev failed\n", __func__); 1044 goto fail; 1045 } 1046 1047 pfd.fd = fd; 1048 pfd.events = POLL_IN; 1049 while (1) { 1050 int i = 0; 1051 char buffer[EV_BUF_LEN]; 1052 ret = poll(&pfd, 1, 1000 * timeout); 1053 1054 if (ret == -1) { 1055 ublk_err("%s: poll inotify failed: %d\n", __func__, ret); 1056 goto rm_watch; 1057 } else if (ret == 0) { 1058 ublk_err("%s: poll inotify timeout\n", __func__); 1059 ret = -ETIMEDOUT; 1060 goto rm_watch; 1061 } 1062 1063 ret = read(fd, buffer, EV_BUF_LEN); 1064 if (ret < 0) { 1065 ublk_err("%s: read inotify fd failed\n", __func__); 1066 goto rm_watch; 1067 } 1068 1069 while (i < ret) { 1070 struct inotify_event *event = (struct inotify_event *)&buffer[i]; 1071 1072 ublk_dbg(UBLK_DBG_DEV, "%s: inotify event %x %s\n", 1073 __func__, event->mask, event->name); 1074 if (event->mask & evt_mask) { 1075 if (!strcmp(event->name, dev_name)) { 1076 ret = 0; 1077 goto rm_watch; 1078 } 1079 } 1080 i += EV_SIZE + event->len; 1081 } 1082 } 1083 rm_watch: 1084 inotify_rm_watch(fd, wd); 1085 fail: 1086 close(fd); 1087 return ret; 1088 } 1089 1090 static int ublk_stop_io_daemon(const struct ublk_dev *dev) 1091 { 1092 int daemon_pid = dev->dev_info.ublksrv_pid; 1093 int dev_id = dev->dev_info.dev_id; 1094 char ublkc[64]; 1095 int ret = 0; 1096 1097 if (daemon_pid < 0) 1098 return 0; 1099 1100 /* daemon may be dead already */ 1101 if (kill(daemon_pid, 0) < 0) 1102 goto wait; 1103 1104 snprintf(ublkc, sizeof(ublkc), "/dev/%s%d", "ublkc", dev_id); 1105 1106 /* ublk char device may be gone already */ 1107 if (access(ublkc, F_OK) != 0) 1108 goto wait; 1109 1110 /* Wait until ublk char device is closed, when the daemon is shutdown */ 1111 ret = wait_ublk_dev(ublkc, IN_CLOSE, 10); 1112 /* double check and since it may be closed before starting inotify */ 1113 if (ret == -ETIMEDOUT) 1114 ret = kill(daemon_pid, 0) < 0; 1115 wait: 1116 waitpid(daemon_pid, NULL, 0); 1117 ublk_dbg(UBLK_DBG_DEV, "%s: pid %d dev_id %d ret %d\n", 1118 __func__, daemon_pid, dev_id, ret); 1119 1120 return ret; 1121 } 1122 1123 static int __cmd_dev_add(const struct dev_ctx *ctx) 1124 { 1125 unsigned nthreads = ctx->nthreads; 1126 unsigned nr_queues = ctx->nr_hw_queues; 1127 const char *tgt_type = ctx->tgt_type; 1128 unsigned depth = ctx->queue_depth; 1129 __u64 features; 1130 const struct ublk_tgt_ops *ops; 1131 struct ublksrv_ctrl_dev_info *info; 1132 struct ublk_dev *dev = NULL; 1133 int dev_id = ctx->dev_id; 1134 int ret, i; 1135 1136 ops = ublk_find_tgt(tgt_type); 1137 if (!ops) { 1138 ublk_err("%s: no such tgt type, type %s\n", 1139 __func__, tgt_type); 1140 ret = -ENODEV; 1141 goto fail; 1142 } 1143 1144 if (nr_queues > UBLK_MAX_QUEUES || depth > UBLK_QUEUE_DEPTH) { 1145 ublk_err("%s: invalid nr_queues or depth queues %u depth %u\n", 1146 __func__, nr_queues, depth); 1147 ret = -EINVAL; 1148 goto fail; 1149 } 1150 1151 /* default to 1:1 threads:queues if nthreads is unspecified */ 1152 if (!nthreads) 1153 nthreads = nr_queues; 1154 1155 if (nthreads > UBLK_MAX_THREADS) { 1156 ublk_err("%s: %u is too many threads (max %u)\n", 1157 __func__, nthreads, UBLK_MAX_THREADS); 1158 ret = -EINVAL; 1159 goto fail; 1160 } 1161 1162 if (nthreads != nr_queues && !ctx->per_io_tasks) { 1163 ublk_err("%s: threads %u must be same as queues %u if " 1164 "not using per_io_tasks\n", 1165 __func__, nthreads, nr_queues); 1166 ret = -EINVAL; 1167 goto fail; 1168 } 1169 1170 dev = ublk_ctrl_init(); 1171 if (!dev) { 1172 ublk_err("%s: can't alloc dev id %d, type %s\n", 1173 __func__, dev_id, tgt_type); 1174 ret = -ENOMEM; 1175 goto fail; 1176 } 1177 1178 /* kernel doesn't support get_features */ 1179 ret = ublk_ctrl_get_features(dev, &features); 1180 if (ret < 0) { 1181 ret = -EINVAL; 1182 goto fail; 1183 } 1184 1185 if (!(features & UBLK_F_CMD_IOCTL_ENCODE)) { 1186 ret = -ENOTSUP; 1187 goto fail; 1188 } 1189 1190 info = &dev->dev_info; 1191 info->dev_id = ctx->dev_id; 1192 info->nr_hw_queues = nr_queues; 1193 info->queue_depth = depth; 1194 info->flags = ctx->flags; 1195 if ((features & UBLK_F_QUIESCE) && 1196 (info->flags & UBLK_F_USER_RECOVERY)) 1197 info->flags |= UBLK_F_QUIESCE; 1198 dev->nthreads = nthreads; 1199 dev->per_io_tasks = ctx->per_io_tasks; 1200 dev->tgt.ops = ops; 1201 dev->tgt.sq_depth = depth; 1202 dev->tgt.cq_depth = depth; 1203 1204 for (i = 0; i < MAX_BACK_FILES; i++) { 1205 if (ctx->files[i]) { 1206 strcpy(dev->tgt.backing_file[i], ctx->files[i]); 1207 dev->tgt.nr_backing_files++; 1208 } 1209 } 1210 1211 if (ctx->recovery) 1212 ret = ublk_ctrl_start_user_recovery(dev); 1213 else 1214 ret = ublk_ctrl_add_dev(dev); 1215 if (ret < 0) { 1216 ublk_err("%s: can't add dev id %d, type %s ret %d\n", 1217 __func__, dev_id, tgt_type, ret); 1218 goto fail; 1219 } 1220 1221 ret = ublk_start_daemon(ctx, dev); 1222 ublk_dbg(UBLK_DBG_DEV, "%s: daemon exit %d\b", ret); 1223 if (ret < 0) 1224 ublk_ctrl_del_dev(dev); 1225 1226 fail: 1227 if (ret < 0) 1228 ublk_send_dev_event(ctx, dev, -1); 1229 if (dev) 1230 ublk_ctrl_deinit(dev); 1231 return ret; 1232 } 1233 1234 static int __cmd_dev_list(struct dev_ctx *ctx); 1235 1236 static int cmd_dev_add(struct dev_ctx *ctx) 1237 { 1238 int res; 1239 1240 if (ctx->fg) 1241 goto run; 1242 1243 ctx->_shmid = shmget(IPC_PRIVATE, sizeof(struct ublk_dev), IPC_CREAT | 0666); 1244 if (ctx->_shmid < 0) { 1245 ublk_err("%s: failed to shmget %s\n", __func__, strerror(errno)); 1246 exit(-1); 1247 } 1248 ctx->shadow_dev = (struct ublk_dev *)shmat(ctx->_shmid, NULL, 0); 1249 if (ctx->shadow_dev == (struct ublk_dev *)-1) { 1250 ublk_err("%s: failed to shmat %s\n", __func__, strerror(errno)); 1251 exit(-1); 1252 } 1253 ctx->_evtfd = eventfd(0, 0); 1254 if (ctx->_evtfd < 0) { 1255 ublk_err("%s: failed to create eventfd %s\n", __func__, strerror(errno)); 1256 exit(-1); 1257 } 1258 1259 res = fork(); 1260 if (res == 0) { 1261 int res2; 1262 1263 setsid(); 1264 res2 = fork(); 1265 if (res2 == 0) { 1266 /* prepare for detaching */ 1267 close(STDIN_FILENO); 1268 close(STDOUT_FILENO); 1269 close(STDERR_FILENO); 1270 run: 1271 res = __cmd_dev_add(ctx); 1272 return res; 1273 } else { 1274 /* detached from the foreground task */ 1275 exit(EXIT_SUCCESS); 1276 } 1277 } else if (res > 0) { 1278 uint64_t id; 1279 int exit_code = EXIT_FAILURE; 1280 1281 res = read(ctx->_evtfd, &id, sizeof(id)); 1282 close(ctx->_evtfd); 1283 if (res == sizeof(id) && id != ERROR_EVTFD_DEVID) { 1284 ctx->dev_id = id - 1; 1285 if (__cmd_dev_list(ctx) >= 0) 1286 exit_code = EXIT_SUCCESS; 1287 } 1288 shmdt(ctx->shadow_dev); 1289 shmctl(ctx->_shmid, IPC_RMID, NULL); 1290 /* wait for child and detach from it */ 1291 wait(NULL); 1292 if (exit_code == EXIT_FAILURE) 1293 ublk_err("%s: command failed\n", __func__); 1294 exit(exit_code); 1295 } else { 1296 exit(EXIT_FAILURE); 1297 } 1298 } 1299 1300 static int __cmd_dev_del(struct dev_ctx *ctx) 1301 { 1302 int number = ctx->dev_id; 1303 struct ublk_dev *dev; 1304 int ret; 1305 1306 dev = ublk_ctrl_init(); 1307 dev->dev_info.dev_id = number; 1308 1309 ret = ublk_ctrl_get_info(dev); 1310 if (ret < 0) 1311 goto fail; 1312 1313 ret = ublk_ctrl_stop_dev(dev); 1314 if (ret < 0) 1315 ublk_err("%s: stop dev %d failed ret %d\n", __func__, number, ret); 1316 1317 ret = ublk_stop_io_daemon(dev); 1318 if (ret < 0) 1319 ublk_err("%s: stop daemon id %d dev %d, ret %d\n", 1320 __func__, dev->dev_info.ublksrv_pid, number, ret); 1321 ublk_ctrl_del_dev(dev); 1322 fail: 1323 ublk_ctrl_deinit(dev); 1324 1325 return (ret >= 0) ? 0 : ret; 1326 } 1327 1328 static int cmd_dev_del(struct dev_ctx *ctx) 1329 { 1330 int i; 1331 1332 if (ctx->dev_id >= 0 || !ctx->all) 1333 return __cmd_dev_del(ctx); 1334 1335 for (i = 0; i < 255; i++) { 1336 ctx->dev_id = i; 1337 __cmd_dev_del(ctx); 1338 } 1339 return 0; 1340 } 1341 1342 static int __cmd_dev_list(struct dev_ctx *ctx) 1343 { 1344 struct ublk_dev *dev = ublk_ctrl_init(); 1345 int ret; 1346 1347 if (!dev) 1348 return -ENODEV; 1349 1350 dev->dev_info.dev_id = ctx->dev_id; 1351 1352 ret = ublk_ctrl_get_info(dev); 1353 if (ret < 0) { 1354 if (ctx->logging) 1355 ublk_err("%s: can't get dev info from %d: %d\n", 1356 __func__, ctx->dev_id, ret); 1357 } else { 1358 if (ctx->shadow_dev) 1359 memcpy(&dev->q, ctx->shadow_dev->q, sizeof(dev->q)); 1360 1361 ublk_ctrl_dump(dev); 1362 } 1363 1364 ublk_ctrl_deinit(dev); 1365 1366 return ret; 1367 } 1368 1369 static int cmd_dev_list(struct dev_ctx *ctx) 1370 { 1371 int i; 1372 1373 if (ctx->dev_id >= 0 || !ctx->all) 1374 return __cmd_dev_list(ctx); 1375 1376 ctx->logging = false; 1377 for (i = 0; i < 255; i++) { 1378 ctx->dev_id = i; 1379 __cmd_dev_list(ctx); 1380 } 1381 return 0; 1382 } 1383 1384 static int cmd_dev_get_features(void) 1385 { 1386 #define const_ilog2(x) (63 - __builtin_clzll(x)) 1387 #define FEAT_NAME(f) [const_ilog2(f)] = #f 1388 static const char *feat_map[] = { 1389 FEAT_NAME(UBLK_F_SUPPORT_ZERO_COPY), 1390 FEAT_NAME(UBLK_F_URING_CMD_COMP_IN_TASK), 1391 FEAT_NAME(UBLK_F_NEED_GET_DATA), 1392 FEAT_NAME(UBLK_F_USER_RECOVERY), 1393 FEAT_NAME(UBLK_F_USER_RECOVERY_REISSUE), 1394 FEAT_NAME(UBLK_F_UNPRIVILEGED_DEV), 1395 FEAT_NAME(UBLK_F_CMD_IOCTL_ENCODE), 1396 FEAT_NAME(UBLK_F_USER_COPY), 1397 FEAT_NAME(UBLK_F_ZONED), 1398 FEAT_NAME(UBLK_F_USER_RECOVERY_FAIL_IO), 1399 FEAT_NAME(UBLK_F_UPDATE_SIZE), 1400 FEAT_NAME(UBLK_F_AUTO_BUF_REG), 1401 FEAT_NAME(UBLK_F_QUIESCE), 1402 FEAT_NAME(UBLK_F_PER_IO_DAEMON), 1403 FEAT_NAME(UBLK_F_BUF_REG_OFF_DAEMON), 1404 }; 1405 struct ublk_dev *dev; 1406 __u64 features = 0; 1407 int ret; 1408 1409 dev = ublk_ctrl_init(); 1410 if (!dev) { 1411 fprintf(stderr, "ublksrv_ctrl_init failed id\n"); 1412 return -EOPNOTSUPP; 1413 } 1414 1415 ret = ublk_ctrl_get_features(dev, &features); 1416 if (!ret) { 1417 int i; 1418 1419 printf("ublk_drv features: 0x%llx\n", features); 1420 1421 for (i = 0; i < sizeof(features) * 8; i++) { 1422 const char *feat; 1423 1424 if (!((1ULL << i) & features)) 1425 continue; 1426 if (i < ARRAY_SIZE(feat_map)) 1427 feat = feat_map[i]; 1428 else 1429 feat = "unknown"; 1430 printf("0x%-16llx: %s\n", 1ULL << i, feat); 1431 } 1432 } 1433 1434 return ret; 1435 } 1436 1437 static int cmd_dev_update_size(struct dev_ctx *ctx) 1438 { 1439 struct ublk_dev *dev = ublk_ctrl_init(); 1440 struct ublk_params p; 1441 int ret = -EINVAL; 1442 1443 if (!dev) 1444 return -ENODEV; 1445 1446 if (ctx->dev_id < 0) { 1447 fprintf(stderr, "device id isn't provided\n"); 1448 goto out; 1449 } 1450 1451 dev->dev_info.dev_id = ctx->dev_id; 1452 ret = ublk_ctrl_get_params(dev, &p); 1453 if (ret < 0) { 1454 ublk_err("failed to get params %d %s\n", ret, strerror(-ret)); 1455 goto out; 1456 } 1457 1458 if (ctx->size & ((1 << p.basic.logical_bs_shift) - 1)) { 1459 ublk_err("size isn't aligned with logical block size\n"); 1460 ret = -EINVAL; 1461 goto out; 1462 } 1463 1464 ret = ublk_ctrl_update_size(dev, ctx->size >> 9); 1465 out: 1466 ublk_ctrl_deinit(dev); 1467 return ret; 1468 } 1469 1470 static int cmd_dev_quiesce(struct dev_ctx *ctx) 1471 { 1472 struct ublk_dev *dev = ublk_ctrl_init(); 1473 int ret = -EINVAL; 1474 1475 if (!dev) 1476 return -ENODEV; 1477 1478 if (ctx->dev_id < 0) { 1479 fprintf(stderr, "device id isn't provided for quiesce\n"); 1480 goto out; 1481 } 1482 dev->dev_info.dev_id = ctx->dev_id; 1483 ret = ublk_ctrl_quiesce_dev(dev, 10000); 1484 1485 out: 1486 ublk_ctrl_deinit(dev); 1487 return ret; 1488 } 1489 1490 static void __cmd_create_help(char *exe, bool recovery) 1491 { 1492 int i; 1493 1494 printf("%s %s -t [null|loop|stripe|fault_inject] [-q nr_queues] [-d depth] [-n dev_id]\n", 1495 exe, recovery ? "recover" : "add"); 1496 printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--auto_zc_fallback] [--debug_mask mask] [-r 0|1 ] [-g]\n"); 1497 printf("\t[-e 0|1 ] [-i 0|1] [--no_ublk_fixed_fd]\n"); 1498 printf("\t[--nthreads threads] [--per_io_tasks]\n"); 1499 printf("\t[target options] [backfile1] [backfile2] ...\n"); 1500 printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n"); 1501 printf("\tdefault: nthreads=nr_queues"); 1502 1503 for (i = 0; i < ARRAY_SIZE(tgt_ops_list); i++) { 1504 const struct ublk_tgt_ops *ops = tgt_ops_list[i]; 1505 1506 if (ops->usage) 1507 ops->usage(ops); 1508 } 1509 } 1510 1511 static void cmd_add_help(char *exe) 1512 { 1513 __cmd_create_help(exe, false); 1514 printf("\n"); 1515 } 1516 1517 static void cmd_recover_help(char *exe) 1518 { 1519 __cmd_create_help(exe, true); 1520 printf("\tPlease provide exact command line for creating this device with real dev_id\n"); 1521 printf("\n"); 1522 } 1523 1524 static int cmd_dev_help(char *exe) 1525 { 1526 cmd_add_help(exe); 1527 cmd_recover_help(exe); 1528 1529 printf("%s del [-n dev_id] -a \n", exe); 1530 printf("\t -a delete all devices -n delete specified device\n\n"); 1531 printf("%s list [-n dev_id] -a \n", exe); 1532 printf("\t -a list all devices, -n list specified device, default -a \n\n"); 1533 printf("%s features\n", exe); 1534 printf("%s update_size -n dev_id -s|--size size_in_bytes \n", exe); 1535 printf("%s quiesce -n dev_id\n", exe); 1536 return 0; 1537 } 1538 1539 int main(int argc, char *argv[]) 1540 { 1541 static const struct option longopts[] = { 1542 { "all", 0, NULL, 'a' }, 1543 { "type", 1, NULL, 't' }, 1544 { "number", 1, NULL, 'n' }, 1545 { "queues", 1, NULL, 'q' }, 1546 { "depth", 1, NULL, 'd' }, 1547 { "debug_mask", 1, NULL, 0 }, 1548 { "quiet", 0, NULL, 0 }, 1549 { "zero_copy", 0, NULL, 'z' }, 1550 { "foreground", 0, NULL, 0 }, 1551 { "recovery", 1, NULL, 'r' }, 1552 { "recovery_fail_io", 1, NULL, 'e'}, 1553 { "recovery_reissue", 1, NULL, 'i'}, 1554 { "get_data", 1, NULL, 'g'}, 1555 { "auto_zc", 0, NULL, 0 }, 1556 { "auto_zc_fallback", 0, NULL, 0 }, 1557 { "size", 1, NULL, 's'}, 1558 { "nthreads", 1, NULL, 0 }, 1559 { "per_io_tasks", 0, NULL, 0 }, 1560 { "no_ublk_fixed_fd", 0, NULL, 0 }, 1561 { 0, 0, 0, 0 } 1562 }; 1563 const struct ublk_tgt_ops *ops = NULL; 1564 int option_idx, opt; 1565 const char *cmd = argv[1]; 1566 struct dev_ctx ctx = { 1567 .queue_depth = 128, 1568 .nr_hw_queues = 2, 1569 .dev_id = -1, 1570 .tgt_type = "unknown", 1571 }; 1572 int ret = -EINVAL, i; 1573 int tgt_argc = 1; 1574 char *tgt_argv[MAX_NR_TGT_ARG] = { NULL }; 1575 int value; 1576 1577 if (argc == 1) 1578 return ret; 1579 1580 opterr = 0; 1581 optind = 2; 1582 while ((opt = getopt_long(argc, argv, "t:n:d:q:r:e:i:s:gaz", 1583 longopts, &option_idx)) != -1) { 1584 switch (opt) { 1585 case 'a': 1586 ctx.all = 1; 1587 break; 1588 case 'n': 1589 ctx.dev_id = strtol(optarg, NULL, 10); 1590 break; 1591 case 't': 1592 if (strlen(optarg) < sizeof(ctx.tgt_type)) 1593 strcpy(ctx.tgt_type, optarg); 1594 break; 1595 case 'q': 1596 ctx.nr_hw_queues = strtol(optarg, NULL, 10); 1597 break; 1598 case 'd': 1599 ctx.queue_depth = strtol(optarg, NULL, 10); 1600 break; 1601 case 'z': 1602 ctx.flags |= UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_USER_COPY; 1603 break; 1604 case 'r': 1605 value = strtol(optarg, NULL, 10); 1606 if (value) 1607 ctx.flags |= UBLK_F_USER_RECOVERY; 1608 break; 1609 case 'e': 1610 value = strtol(optarg, NULL, 10); 1611 if (value) 1612 ctx.flags |= UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_FAIL_IO; 1613 break; 1614 case 'i': 1615 value = strtol(optarg, NULL, 10); 1616 if (value) 1617 ctx.flags |= UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_REISSUE; 1618 break; 1619 case 'g': 1620 ctx.flags |= UBLK_F_NEED_GET_DATA; 1621 break; 1622 case 's': 1623 ctx.size = strtoull(optarg, NULL, 10); 1624 break; 1625 case 0: 1626 if (!strcmp(longopts[option_idx].name, "debug_mask")) 1627 ublk_dbg_mask = strtol(optarg, NULL, 16); 1628 if (!strcmp(longopts[option_idx].name, "quiet")) 1629 ublk_dbg_mask = 0; 1630 if (!strcmp(longopts[option_idx].name, "foreground")) 1631 ctx.fg = 1; 1632 if (!strcmp(longopts[option_idx].name, "auto_zc")) 1633 ctx.flags |= UBLK_F_AUTO_BUF_REG; 1634 if (!strcmp(longopts[option_idx].name, "auto_zc_fallback")) 1635 ctx.auto_zc_fallback = 1; 1636 if (!strcmp(longopts[option_idx].name, "nthreads")) 1637 ctx.nthreads = strtol(optarg, NULL, 10); 1638 if (!strcmp(longopts[option_idx].name, "per_io_tasks")) 1639 ctx.per_io_tasks = 1; 1640 if (!strcmp(longopts[option_idx].name, "no_ublk_fixed_fd")) 1641 ctx.no_ublk_fixed_fd = 1; 1642 break; 1643 case '?': 1644 /* 1645 * target requires every option must have argument 1646 */ 1647 if (argv[optind][0] == '-' || argv[optind - 1][0] != '-') { 1648 fprintf(stderr, "every target option requires argument: %s %s\n", 1649 argv[optind - 1], argv[optind]); 1650 exit(EXIT_FAILURE); 1651 } 1652 1653 if (tgt_argc < (MAX_NR_TGT_ARG - 1) / 2) { 1654 tgt_argv[tgt_argc++] = argv[optind - 1]; 1655 tgt_argv[tgt_argc++] = argv[optind]; 1656 } else { 1657 fprintf(stderr, "too many target options\n"); 1658 exit(EXIT_FAILURE); 1659 } 1660 optind += 1; 1661 break; 1662 } 1663 } 1664 1665 /* auto_zc_fallback depends on F_AUTO_BUF_REG & F_SUPPORT_ZERO_COPY */ 1666 if (ctx.auto_zc_fallback && 1667 !((ctx.flags & UBLK_F_AUTO_BUF_REG) && 1668 (ctx.flags & UBLK_F_SUPPORT_ZERO_COPY))) { 1669 ublk_err("%s: auto_zc_fallback is set but neither " 1670 "F_AUTO_BUF_REG nor F_SUPPORT_ZERO_COPY is enabled\n", 1671 __func__); 1672 return -EINVAL; 1673 } 1674 1675 i = optind; 1676 while (i < argc && ctx.nr_files < MAX_BACK_FILES) { 1677 ctx.files[ctx.nr_files++] = argv[i++]; 1678 } 1679 1680 ops = ublk_find_tgt(ctx.tgt_type); 1681 if (ops && ops->parse_cmd_line) { 1682 optind = 0; 1683 1684 tgt_argv[0] = ctx.tgt_type; 1685 ops->parse_cmd_line(&ctx, tgt_argc, tgt_argv); 1686 } 1687 1688 if (!strcmp(cmd, "add")) 1689 ret = cmd_dev_add(&ctx); 1690 else if (!strcmp(cmd, "recover")) { 1691 if (ctx.dev_id < 0) { 1692 fprintf(stderr, "device id isn't provided for recovering\n"); 1693 ret = -EINVAL; 1694 } else { 1695 ctx.recovery = 1; 1696 ret = cmd_dev_add(&ctx); 1697 } 1698 } else if (!strcmp(cmd, "del")) 1699 ret = cmd_dev_del(&ctx); 1700 else if (!strcmp(cmd, "list")) { 1701 ctx.all = 1; 1702 ret = cmd_dev_list(&ctx); 1703 } else if (!strcmp(cmd, "help")) 1704 ret = cmd_dev_help(argv[0]); 1705 else if (!strcmp(cmd, "features")) 1706 ret = cmd_dev_get_features(); 1707 else if (!strcmp(cmd, "update_size")) 1708 ret = cmd_dev_update_size(&ctx); 1709 else if (!strcmp(cmd, "quiesce")) 1710 ret = cmd_dev_quiesce(&ctx); 1711 else 1712 cmd_dev_help(argv[0]); 1713 1714 return ret; 1715 } 1716