1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Description: uring_cmd based ublk 4 */ 5 6 #include "kublk.h" 7 8 #define MAX_NR_TGT_ARG 64 9 10 unsigned int ublk_dbg_mask = UBLK_LOG; 11 static const struct ublk_tgt_ops *tgt_ops_list[] = { 12 &null_tgt_ops, 13 &loop_tgt_ops, 14 &stripe_tgt_ops, 15 &fault_inject_tgt_ops, 16 }; 17 18 static const struct ublk_tgt_ops *ublk_find_tgt(const char *name) 19 { 20 int i; 21 22 if (name == NULL) 23 return NULL; 24 25 for (i = 0; i < ARRAY_SIZE(tgt_ops_list); i++) 26 if (strcmp(tgt_ops_list[i]->name, name) == 0) 27 return tgt_ops_list[i]; 28 return NULL; 29 } 30 31 static inline int ublk_setup_ring(struct io_uring *r, int depth, 32 int cq_depth, unsigned flags) 33 { 34 struct io_uring_params p; 35 36 memset(&p, 0, sizeof(p)); 37 p.flags = flags | IORING_SETUP_CQSIZE; 38 p.cq_entries = cq_depth; 39 40 return io_uring_queue_init_params(depth, r, &p); 41 } 42 43 static void ublk_ctrl_init_cmd(struct ublk_dev *dev, 44 struct io_uring_sqe *sqe, 45 struct ublk_ctrl_cmd_data *data) 46 { 47 struct ublksrv_ctrl_dev_info *info = &dev->dev_info; 48 struct ublksrv_ctrl_cmd *cmd = (struct ublksrv_ctrl_cmd *)ublk_get_sqe_cmd(sqe); 49 50 sqe->fd = dev->ctrl_fd; 51 sqe->opcode = IORING_OP_URING_CMD; 52 sqe->ioprio = 0; 53 54 if (data->flags & CTRL_CMD_HAS_BUF) { 55 cmd->addr = data->addr; 56 cmd->len = data->len; 57 } 58 59 if (data->flags & CTRL_CMD_HAS_DATA) 60 cmd->data[0] = data->data[0]; 61 62 cmd->dev_id = info->dev_id; 63 cmd->queue_id = -1; 64 65 ublk_set_sqe_cmd_op(sqe, data->cmd_op); 66 67 io_uring_sqe_set_data(sqe, cmd); 68 } 69 70 static int __ublk_ctrl_cmd(struct ublk_dev *dev, 71 struct ublk_ctrl_cmd_data *data) 72 { 73 struct io_uring_sqe *sqe; 74 struct io_uring_cqe *cqe; 75 int ret = -EINVAL; 76 77 sqe = io_uring_get_sqe(&dev->ring); 78 if (!sqe) { 79 ublk_err("%s: can't get sqe ret %d\n", __func__, ret); 80 return ret; 81 } 82 83 ublk_ctrl_init_cmd(dev, sqe, data); 84 85 ret = io_uring_submit(&dev->ring); 86 if (ret < 0) { 87 ublk_err("uring submit ret %d\n", ret); 88 return ret; 89 } 90 91 ret = io_uring_wait_cqe(&dev->ring, &cqe); 92 if (ret < 0) { 93 ublk_err("wait cqe: %s\n", strerror(-ret)); 94 return ret; 95 } 96 io_uring_cqe_seen(&dev->ring, cqe); 97 98 return cqe->res; 99 } 100 101 static int ublk_ctrl_stop_dev(struct ublk_dev *dev) 102 { 103 struct ublk_ctrl_cmd_data data = { 104 .cmd_op = UBLK_U_CMD_STOP_DEV, 105 }; 106 107 return __ublk_ctrl_cmd(dev, &data); 108 } 109 110 static int ublk_ctrl_start_dev(struct ublk_dev *dev, 111 int daemon_pid) 112 { 113 struct ublk_ctrl_cmd_data data = { 114 .cmd_op = UBLK_U_CMD_START_DEV, 115 .flags = CTRL_CMD_HAS_DATA, 116 }; 117 118 dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid; 119 120 return __ublk_ctrl_cmd(dev, &data); 121 } 122 123 static int ublk_ctrl_start_user_recovery(struct ublk_dev *dev) 124 { 125 struct ublk_ctrl_cmd_data data = { 126 .cmd_op = UBLK_U_CMD_START_USER_RECOVERY, 127 }; 128 129 return __ublk_ctrl_cmd(dev, &data); 130 } 131 132 static int ublk_ctrl_end_user_recovery(struct ublk_dev *dev, int daemon_pid) 133 { 134 struct ublk_ctrl_cmd_data data = { 135 .cmd_op = UBLK_U_CMD_END_USER_RECOVERY, 136 .flags = CTRL_CMD_HAS_DATA, 137 }; 138 139 dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid; 140 141 return __ublk_ctrl_cmd(dev, &data); 142 } 143 144 static int ublk_ctrl_add_dev(struct ublk_dev *dev) 145 { 146 struct ublk_ctrl_cmd_data data = { 147 .cmd_op = UBLK_U_CMD_ADD_DEV, 148 .flags = CTRL_CMD_HAS_BUF, 149 .addr = (__u64) (uintptr_t) &dev->dev_info, 150 .len = sizeof(struct ublksrv_ctrl_dev_info), 151 }; 152 153 return __ublk_ctrl_cmd(dev, &data); 154 } 155 156 static int ublk_ctrl_del_dev(struct ublk_dev *dev) 157 { 158 struct ublk_ctrl_cmd_data data = { 159 .cmd_op = UBLK_U_CMD_DEL_DEV, 160 .flags = 0, 161 }; 162 163 return __ublk_ctrl_cmd(dev, &data); 164 } 165 166 static int ublk_ctrl_get_info(struct ublk_dev *dev) 167 { 168 struct ublk_ctrl_cmd_data data = { 169 .cmd_op = UBLK_U_CMD_GET_DEV_INFO, 170 .flags = CTRL_CMD_HAS_BUF, 171 .addr = (__u64) (uintptr_t) &dev->dev_info, 172 .len = sizeof(struct ublksrv_ctrl_dev_info), 173 }; 174 175 return __ublk_ctrl_cmd(dev, &data); 176 } 177 178 static int ublk_ctrl_set_params(struct ublk_dev *dev, 179 struct ublk_params *params) 180 { 181 struct ublk_ctrl_cmd_data data = { 182 .cmd_op = UBLK_U_CMD_SET_PARAMS, 183 .flags = CTRL_CMD_HAS_BUF, 184 .addr = (__u64) (uintptr_t) params, 185 .len = sizeof(*params), 186 }; 187 params->len = sizeof(*params); 188 return __ublk_ctrl_cmd(dev, &data); 189 } 190 191 static int ublk_ctrl_get_params(struct ublk_dev *dev, 192 struct ublk_params *params) 193 { 194 struct ublk_ctrl_cmd_data data = { 195 .cmd_op = UBLK_U_CMD_GET_PARAMS, 196 .flags = CTRL_CMD_HAS_BUF, 197 .addr = (__u64)params, 198 .len = sizeof(*params), 199 }; 200 201 params->len = sizeof(*params); 202 203 return __ublk_ctrl_cmd(dev, &data); 204 } 205 206 static int ublk_ctrl_get_features(struct ublk_dev *dev, 207 __u64 *features) 208 { 209 struct ublk_ctrl_cmd_data data = { 210 .cmd_op = UBLK_U_CMD_GET_FEATURES, 211 .flags = CTRL_CMD_HAS_BUF, 212 .addr = (__u64) (uintptr_t) features, 213 .len = sizeof(*features), 214 }; 215 216 return __ublk_ctrl_cmd(dev, &data); 217 } 218 219 static int ublk_ctrl_update_size(struct ublk_dev *dev, 220 __u64 nr_sects) 221 { 222 struct ublk_ctrl_cmd_data data = { 223 .cmd_op = UBLK_U_CMD_UPDATE_SIZE, 224 .flags = CTRL_CMD_HAS_DATA, 225 }; 226 227 data.data[0] = nr_sects; 228 return __ublk_ctrl_cmd(dev, &data); 229 } 230 231 static int ublk_ctrl_quiesce_dev(struct ublk_dev *dev, 232 unsigned int timeout_ms) 233 { 234 struct ublk_ctrl_cmd_data data = { 235 .cmd_op = UBLK_U_CMD_QUIESCE_DEV, 236 .flags = CTRL_CMD_HAS_DATA, 237 }; 238 239 data.data[0] = timeout_ms; 240 return __ublk_ctrl_cmd(dev, &data); 241 } 242 243 static const char *ublk_dev_state_desc(struct ublk_dev *dev) 244 { 245 switch (dev->dev_info.state) { 246 case UBLK_S_DEV_DEAD: 247 return "DEAD"; 248 case UBLK_S_DEV_LIVE: 249 return "LIVE"; 250 case UBLK_S_DEV_QUIESCED: 251 return "QUIESCED"; 252 default: 253 return "UNKNOWN"; 254 }; 255 } 256 257 static void ublk_print_cpu_set(const cpu_set_t *set, char *buf, unsigned len) 258 { 259 unsigned done = 0; 260 int i; 261 262 for (i = 0; i < CPU_SETSIZE; i++) { 263 if (CPU_ISSET(i, set)) 264 done += snprintf(&buf[done], len - done, "%d ", i); 265 } 266 } 267 268 static void ublk_adjust_affinity(cpu_set_t *set) 269 { 270 int j, updated = 0; 271 272 /* 273 * Just keep the 1st CPU now. 274 * 275 * In future, auto affinity selection can be tried. 276 */ 277 for (j = 0; j < CPU_SETSIZE; j++) { 278 if (CPU_ISSET(j, set)) { 279 if (!updated) { 280 updated = 1; 281 continue; 282 } 283 CPU_CLR(j, set); 284 } 285 } 286 } 287 288 /* Caller must free the allocated buffer */ 289 static int ublk_ctrl_get_affinity(struct ublk_dev *ctrl_dev, cpu_set_t **ptr_buf) 290 { 291 struct ublk_ctrl_cmd_data data = { 292 .cmd_op = UBLK_U_CMD_GET_QUEUE_AFFINITY, 293 .flags = CTRL_CMD_HAS_DATA | CTRL_CMD_HAS_BUF, 294 }; 295 cpu_set_t *buf; 296 int i, ret; 297 298 buf = malloc(sizeof(cpu_set_t) * ctrl_dev->dev_info.nr_hw_queues); 299 if (!buf) 300 return -ENOMEM; 301 302 for (i = 0; i < ctrl_dev->dev_info.nr_hw_queues; i++) { 303 data.data[0] = i; 304 data.len = sizeof(cpu_set_t); 305 data.addr = (__u64)&buf[i]; 306 307 ret = __ublk_ctrl_cmd(ctrl_dev, &data); 308 if (ret < 0) { 309 free(buf); 310 return ret; 311 } 312 ublk_adjust_affinity(&buf[i]); 313 } 314 315 *ptr_buf = buf; 316 return 0; 317 } 318 319 static void ublk_ctrl_dump(struct ublk_dev *dev) 320 { 321 struct ublksrv_ctrl_dev_info *info = &dev->dev_info; 322 struct ublk_params p; 323 cpu_set_t *affinity; 324 int ret; 325 326 ret = ublk_ctrl_get_params(dev, &p); 327 if (ret < 0) { 328 ublk_err("failed to get params %d %s\n", ret, strerror(-ret)); 329 return; 330 } 331 332 ret = ublk_ctrl_get_affinity(dev, &affinity); 333 if (ret < 0) { 334 ublk_err("failed to get affinity %m\n"); 335 return; 336 } 337 338 ublk_log("dev id %d: nr_hw_queues %d queue_depth %d block size %d dev_capacity %lld\n", 339 info->dev_id, info->nr_hw_queues, info->queue_depth, 340 1 << p.basic.logical_bs_shift, p.basic.dev_sectors); 341 ublk_log("\tmax rq size %d daemon pid %d flags 0x%llx state %s\n", 342 info->max_io_buf_bytes, info->ublksrv_pid, info->flags, 343 ublk_dev_state_desc(dev)); 344 345 if (affinity) { 346 char buf[512]; 347 int i; 348 349 for (i = 0; i < info->nr_hw_queues; i++) { 350 ublk_print_cpu_set(&affinity[i], buf, sizeof(buf)); 351 printf("\tqueue %u: affinity(%s)\n", 352 i, buf); 353 } 354 free(affinity); 355 } 356 357 fflush(stdout); 358 } 359 360 static void ublk_ctrl_deinit(struct ublk_dev *dev) 361 { 362 close(dev->ctrl_fd); 363 free(dev); 364 } 365 366 static struct ublk_dev *ublk_ctrl_init(void) 367 { 368 struct ublk_dev *dev = (struct ublk_dev *)calloc(1, sizeof(*dev)); 369 struct ublksrv_ctrl_dev_info *info = &dev->dev_info; 370 int ret; 371 372 dev->ctrl_fd = open(CTRL_DEV, O_RDWR); 373 if (dev->ctrl_fd < 0) { 374 free(dev); 375 return NULL; 376 } 377 378 info->max_io_buf_bytes = UBLK_IO_MAX_BYTES; 379 380 ret = ublk_setup_ring(&dev->ring, UBLK_CTRL_RING_DEPTH, 381 UBLK_CTRL_RING_DEPTH, IORING_SETUP_SQE128); 382 if (ret < 0) { 383 ublk_err("queue_init: %s\n", strerror(-ret)); 384 free(dev); 385 return NULL; 386 } 387 dev->nr_fds = 1; 388 389 return dev; 390 } 391 392 static int __ublk_queue_cmd_buf_sz(unsigned depth) 393 { 394 int size = depth * sizeof(struct ublksrv_io_desc); 395 unsigned int page_sz = getpagesize(); 396 397 return round_up(size, page_sz); 398 } 399 400 static int ublk_queue_max_cmd_buf_sz(void) 401 { 402 return __ublk_queue_cmd_buf_sz(UBLK_MAX_QUEUE_DEPTH); 403 } 404 405 static int ublk_queue_cmd_buf_sz(struct ublk_queue *q) 406 { 407 return __ublk_queue_cmd_buf_sz(q->q_depth); 408 } 409 410 static void ublk_queue_deinit(struct ublk_queue *q) 411 { 412 int i; 413 int nr_ios = q->q_depth; 414 415 if (q->io_cmd_buf) 416 munmap(q->io_cmd_buf, ublk_queue_cmd_buf_sz(q)); 417 418 for (i = 0; i < nr_ios; i++) 419 free(q->ios[i].buf_addr); 420 } 421 422 static void ublk_thread_deinit(struct ublk_thread *t) 423 { 424 io_uring_unregister_buffers(&t->ring); 425 426 io_uring_unregister_ring_fd(&t->ring); 427 428 if (t->ring.ring_fd > 0) { 429 io_uring_unregister_files(&t->ring); 430 close(t->ring.ring_fd); 431 t->ring.ring_fd = -1; 432 } 433 } 434 435 static int ublk_queue_init(struct ublk_queue *q, unsigned long long extra_flags) 436 { 437 struct ublk_dev *dev = q->dev; 438 int depth = dev->dev_info.queue_depth; 439 int i; 440 int cmd_buf_size, io_buf_size; 441 unsigned long off; 442 443 q->tgt_ops = dev->tgt.ops; 444 q->flags = 0; 445 q->q_depth = depth; 446 q->flags = dev->dev_info.flags; 447 q->flags |= extra_flags; 448 449 /* Cache fd in queue for fast path access */ 450 q->ublk_fd = dev->fds[0]; 451 452 cmd_buf_size = ublk_queue_cmd_buf_sz(q); 453 off = UBLKSRV_CMD_BUF_OFFSET + q->q_id * ublk_queue_max_cmd_buf_sz(); 454 q->io_cmd_buf = mmap(0, cmd_buf_size, PROT_READ, 455 MAP_SHARED | MAP_POPULATE, dev->fds[0], off); 456 if (q->io_cmd_buf == MAP_FAILED) { 457 ublk_err("ublk dev %d queue %d map io_cmd_buf failed %m\n", 458 q->dev->dev_info.dev_id, q->q_id); 459 goto fail; 460 } 461 462 io_buf_size = dev->dev_info.max_io_buf_bytes; 463 for (i = 0; i < q->q_depth; i++) { 464 q->ios[i].buf_addr = NULL; 465 q->ios[i].flags = UBLKS_IO_NEED_FETCH_RQ | UBLKS_IO_FREE; 466 q->ios[i].tag = i; 467 468 if (ublk_queue_no_buf(q)) 469 continue; 470 471 if (posix_memalign((void **)&q->ios[i].buf_addr, 472 getpagesize(), io_buf_size)) { 473 ublk_err("ublk dev %d queue %d io %d posix_memalign failed %m\n", 474 dev->dev_info.dev_id, q->q_id, i); 475 goto fail; 476 } 477 } 478 479 return 0; 480 fail: 481 ublk_queue_deinit(q); 482 ublk_err("ublk dev %d queue %d failed\n", 483 dev->dev_info.dev_id, q->q_id); 484 return -ENOMEM; 485 } 486 487 static int ublk_thread_init(struct ublk_thread *t, unsigned long long extra_flags) 488 { 489 struct ublk_dev *dev = t->dev; 490 unsigned long long flags = dev->dev_info.flags | extra_flags; 491 int ring_depth = dev->tgt.sq_depth, cq_depth = dev->tgt.cq_depth; 492 int ret; 493 494 ret = ublk_setup_ring(&t->ring, ring_depth, cq_depth, 495 IORING_SETUP_COOP_TASKRUN | 496 IORING_SETUP_SINGLE_ISSUER | 497 IORING_SETUP_DEFER_TASKRUN); 498 if (ret < 0) { 499 ublk_err("ublk dev %d thread %d setup io_uring failed %d\n", 500 dev->dev_info.dev_id, t->idx, ret); 501 goto fail; 502 } 503 504 if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) { 505 unsigned nr_ios = dev->dev_info.queue_depth * dev->dev_info.nr_hw_queues; 506 unsigned max_nr_ios_per_thread = nr_ios / dev->nthreads; 507 max_nr_ios_per_thread += !!(nr_ios % dev->nthreads); 508 ret = io_uring_register_buffers_sparse( 509 &t->ring, max_nr_ios_per_thread); 510 if (ret) { 511 ublk_err("ublk dev %d thread %d register spare buffers failed %d", 512 dev->dev_info.dev_id, t->idx, ret); 513 goto fail; 514 } 515 } 516 517 io_uring_register_ring_fd(&t->ring); 518 519 if (flags & UBLKS_Q_NO_UBLK_FIXED_FD) { 520 /* Register only backing files starting from index 1, exclude ublk control device */ 521 if (dev->nr_fds > 1) { 522 ret = io_uring_register_files(&t->ring, &dev->fds[1], dev->nr_fds - 1); 523 } else { 524 /* No backing files to register, skip file registration */ 525 ret = 0; 526 } 527 } else { 528 ret = io_uring_register_files(&t->ring, dev->fds, dev->nr_fds); 529 } 530 if (ret) { 531 ublk_err("ublk dev %d thread %d register files failed %d\n", 532 t->dev->dev_info.dev_id, t->idx, ret); 533 goto fail; 534 } 535 536 return 0; 537 fail: 538 ublk_thread_deinit(t); 539 ublk_err("ublk dev %d thread %d init failed\n", 540 dev->dev_info.dev_id, t->idx); 541 return -ENOMEM; 542 } 543 544 #define WAIT_USEC 100000 545 #define MAX_WAIT_USEC (3 * 1000000) 546 static int ublk_dev_prep(const struct dev_ctx *ctx, struct ublk_dev *dev) 547 { 548 int dev_id = dev->dev_info.dev_id; 549 unsigned int wait_usec = 0; 550 int ret = 0, fd = -1; 551 char buf[64]; 552 553 snprintf(buf, 64, "%s%d", UBLKC_DEV, dev_id); 554 555 while (wait_usec < MAX_WAIT_USEC) { 556 fd = open(buf, O_RDWR); 557 if (fd >= 0) 558 break; 559 usleep(WAIT_USEC); 560 wait_usec += WAIT_USEC; 561 } 562 if (fd < 0) { 563 ublk_err("can't open %s %s\n", buf, strerror(errno)); 564 return -1; 565 } 566 567 dev->fds[0] = fd; 568 if (dev->tgt.ops->init_tgt) 569 ret = dev->tgt.ops->init_tgt(ctx, dev); 570 if (ret) 571 close(dev->fds[0]); 572 return ret; 573 } 574 575 static void ublk_dev_unprep(struct ublk_dev *dev) 576 { 577 if (dev->tgt.ops->deinit_tgt) 578 dev->tgt.ops->deinit_tgt(dev); 579 close(dev->fds[0]); 580 } 581 582 static void ublk_set_auto_buf_reg(const struct ublk_queue *q, 583 struct io_uring_sqe *sqe, 584 unsigned short tag) 585 { 586 struct ublk_auto_buf_reg buf = {}; 587 588 if (q->tgt_ops->buf_index) 589 buf.index = q->tgt_ops->buf_index(q, tag); 590 else 591 buf.index = q->ios[tag].buf_index; 592 593 if (ublk_queue_auto_zc_fallback(q)) 594 buf.flags = UBLK_AUTO_BUF_REG_FALLBACK; 595 596 sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf); 597 } 598 599 int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io) 600 { 601 struct ublk_queue *q = ublk_io_to_queue(io); 602 struct ublksrv_io_cmd *cmd; 603 struct io_uring_sqe *sqe[1]; 604 unsigned int cmd_op = 0; 605 __u64 user_data; 606 607 /* only freed io can be issued */ 608 if (!(io->flags & UBLKS_IO_FREE)) 609 return 0; 610 611 /* 612 * we issue because we need either fetching or committing or 613 * getting data 614 */ 615 if (!(io->flags & 616 (UBLKS_IO_NEED_FETCH_RQ | UBLKS_IO_NEED_COMMIT_RQ_COMP | UBLKS_IO_NEED_GET_DATA))) 617 return 0; 618 619 if (io->flags & UBLKS_IO_NEED_GET_DATA) 620 cmd_op = UBLK_U_IO_NEED_GET_DATA; 621 else if (io->flags & UBLKS_IO_NEED_COMMIT_RQ_COMP) 622 cmd_op = UBLK_U_IO_COMMIT_AND_FETCH_REQ; 623 else if (io->flags & UBLKS_IO_NEED_FETCH_RQ) 624 cmd_op = UBLK_U_IO_FETCH_REQ; 625 626 if (io_uring_sq_space_left(&t->ring) < 1) 627 io_uring_submit(&t->ring); 628 629 ublk_io_alloc_sqes(t, sqe, 1); 630 if (!sqe[0]) { 631 ublk_err("%s: run out of sqe. thread %u, tag %d\n", 632 __func__, t->idx, io->tag); 633 return -1; 634 } 635 636 cmd = (struct ublksrv_io_cmd *)ublk_get_sqe_cmd(sqe[0]); 637 638 if (cmd_op == UBLK_U_IO_COMMIT_AND_FETCH_REQ) 639 cmd->result = io->result; 640 641 /* These fields should be written once, never change */ 642 ublk_set_sqe_cmd_op(sqe[0], cmd_op); 643 sqe[0]->fd = ublk_get_registered_fd(q, 0); /* dev->fds[0] */ 644 sqe[0]->opcode = IORING_OP_URING_CMD; 645 if (q->flags & UBLKS_Q_NO_UBLK_FIXED_FD) 646 sqe[0]->flags = 0; /* Use raw FD, not fixed file */ 647 else 648 sqe[0]->flags = IOSQE_FIXED_FILE; 649 sqe[0]->rw_flags = 0; 650 cmd->tag = io->tag; 651 cmd->q_id = q->q_id; 652 if (!ublk_queue_no_buf(q)) 653 cmd->addr = (__u64) (uintptr_t) io->buf_addr; 654 else 655 cmd->addr = 0; 656 657 if (ublk_queue_use_auto_zc(q)) 658 ublk_set_auto_buf_reg(q, sqe[0], io->tag); 659 660 user_data = build_user_data(io->tag, _IOC_NR(cmd_op), 0, q->q_id, 0); 661 io_uring_sqe_set_data64(sqe[0], user_data); 662 663 io->flags = 0; 664 665 t->cmd_inflight += 1; 666 667 ublk_dbg(UBLK_DBG_IO_CMD, "%s: (thread %u qid %d tag %u cmd_op %u) iof %x stopping %d\n", 668 __func__, t->idx, q->q_id, io->tag, cmd_op, 669 io->flags, !!(t->state & UBLKS_T_STOPPING)); 670 return 1; 671 } 672 673 static void ublk_submit_fetch_commands(struct ublk_thread *t) 674 { 675 struct ublk_queue *q; 676 struct ublk_io *io; 677 int i = 0, j = 0; 678 679 if (t->dev->per_io_tasks) { 680 /* 681 * Lexicographically order all the (qid,tag) pairs, with 682 * qid taking priority (so (1,0) > (0,1)). Then make 683 * this thread the daemon for every Nth entry in this 684 * list (N is the number of threads), starting at this 685 * thread's index. This ensures that each queue is 686 * handled by as many ublk server threads as possible, 687 * so that load that is concentrated on one or a few 688 * queues can make use of all ublk server threads. 689 */ 690 const struct ublksrv_ctrl_dev_info *dinfo = &t->dev->dev_info; 691 int nr_ios = dinfo->nr_hw_queues * dinfo->queue_depth; 692 for (i = t->idx; i < nr_ios; i += t->dev->nthreads) { 693 int q_id = i / dinfo->queue_depth; 694 int tag = i % dinfo->queue_depth; 695 q = &t->dev->q[q_id]; 696 io = &q->ios[tag]; 697 io->buf_index = j++; 698 ublk_queue_io_cmd(t, io); 699 } 700 } else { 701 /* 702 * Service exclusively the queue whose q_id matches our 703 * thread index. 704 */ 705 struct ublk_queue *q = &t->dev->q[t->idx]; 706 for (i = 0; i < q->q_depth; i++) { 707 io = &q->ios[i]; 708 io->buf_index = i; 709 ublk_queue_io_cmd(t, io); 710 } 711 } 712 } 713 714 static int ublk_thread_is_idle(struct ublk_thread *t) 715 { 716 return !io_uring_sq_ready(&t->ring) && !t->io_inflight; 717 } 718 719 static int ublk_thread_is_done(struct ublk_thread *t) 720 { 721 return (t->state & UBLKS_T_STOPPING) && ublk_thread_is_idle(t); 722 } 723 724 static inline void ublksrv_handle_tgt_cqe(struct ublk_thread *t, 725 struct ublk_queue *q, 726 struct io_uring_cqe *cqe) 727 { 728 if (cqe->res < 0 && cqe->res != -EAGAIN) 729 ublk_err("%s: failed tgt io: res %d qid %u tag %u, cmd_op %u\n", 730 __func__, cqe->res, q->q_id, 731 user_data_to_tag(cqe->user_data), 732 user_data_to_op(cqe->user_data)); 733 734 if (q->tgt_ops->tgt_io_done) 735 q->tgt_ops->tgt_io_done(t, q, cqe); 736 } 737 738 static void ublk_handle_uring_cmd(struct ublk_thread *t, 739 struct ublk_queue *q, 740 const struct io_uring_cqe *cqe) 741 { 742 int fetch = (cqe->res != UBLK_IO_RES_ABORT) && 743 !(t->state & UBLKS_T_STOPPING); 744 unsigned tag = user_data_to_tag(cqe->user_data); 745 struct ublk_io *io = &q->ios[tag]; 746 747 if (!fetch) { 748 t->state |= UBLKS_T_STOPPING; 749 io->flags &= ~UBLKS_IO_NEED_FETCH_RQ; 750 } 751 752 if (cqe->res == UBLK_IO_RES_OK) { 753 assert(tag < q->q_depth); 754 if (q->tgt_ops->queue_io) 755 q->tgt_ops->queue_io(t, q, tag); 756 } else if (cqe->res == UBLK_IO_RES_NEED_GET_DATA) { 757 io->flags |= UBLKS_IO_NEED_GET_DATA | UBLKS_IO_FREE; 758 ublk_queue_io_cmd(t, io); 759 } else { 760 /* 761 * COMMIT_REQ will be completed immediately since no fetching 762 * piggyback is required. 763 * 764 * Marking IO_FREE only, then this io won't be issued since 765 * we only issue io with (UBLKS_IO_FREE | UBLKSRV_NEED_*) 766 * 767 * */ 768 io->flags = UBLKS_IO_FREE; 769 } 770 } 771 772 static void ublk_handle_cqe(struct ublk_thread *t, 773 struct io_uring_cqe *cqe, void *data) 774 { 775 struct ublk_dev *dev = t->dev; 776 unsigned q_id = user_data_to_q_id(cqe->user_data); 777 struct ublk_queue *q = &dev->q[q_id]; 778 unsigned cmd_op = user_data_to_op(cqe->user_data); 779 780 if (cqe->res < 0 && cqe->res != -ENODEV) 781 ublk_err("%s: res %d userdata %llx queue state %x\n", __func__, 782 cqe->res, cqe->user_data, q->flags); 783 784 ublk_dbg(UBLK_DBG_IO_CMD, "%s: res %d (qid %d tag %u cmd_op %u target %d/%d) stopping %d\n", 785 __func__, cqe->res, q->q_id, user_data_to_tag(cqe->user_data), 786 cmd_op, is_target_io(cqe->user_data), 787 user_data_to_tgt_data(cqe->user_data), 788 (t->state & UBLKS_T_STOPPING)); 789 790 /* Don't retrieve io in case of target io */ 791 if (is_target_io(cqe->user_data)) { 792 ublksrv_handle_tgt_cqe(t, q, cqe); 793 return; 794 } 795 796 t->cmd_inflight--; 797 798 ublk_handle_uring_cmd(t, q, cqe); 799 } 800 801 static int ublk_reap_events_uring(struct ublk_thread *t) 802 { 803 struct io_uring_cqe *cqe; 804 unsigned head; 805 int count = 0; 806 807 io_uring_for_each_cqe(&t->ring, head, cqe) { 808 ublk_handle_cqe(t, cqe, NULL); 809 count += 1; 810 } 811 io_uring_cq_advance(&t->ring, count); 812 813 return count; 814 } 815 816 static int ublk_process_io(struct ublk_thread *t) 817 { 818 int ret, reapped; 819 820 ublk_dbg(UBLK_DBG_THREAD, "dev%d-t%u: to_submit %d inflight cmd %u stopping %d\n", 821 t->dev->dev_info.dev_id, 822 t->idx, io_uring_sq_ready(&t->ring), 823 t->cmd_inflight, 824 (t->state & UBLKS_T_STOPPING)); 825 826 if (ublk_thread_is_done(t)) 827 return -ENODEV; 828 829 ret = io_uring_submit_and_wait(&t->ring, 1); 830 reapped = ublk_reap_events_uring(t); 831 832 ublk_dbg(UBLK_DBG_THREAD, "submit result %d, reapped %d stop %d idle %d\n", 833 ret, reapped, (t->state & UBLKS_T_STOPPING), 834 (t->state & UBLKS_T_IDLE)); 835 836 return reapped; 837 } 838 839 struct ublk_thread_info { 840 struct ublk_dev *dev; 841 pthread_t thread; 842 unsigned idx; 843 sem_t *ready; 844 cpu_set_t *affinity; 845 unsigned long long extra_flags; 846 }; 847 848 static void ublk_thread_set_sched_affinity(const struct ublk_thread_info *info) 849 { 850 if (pthread_setaffinity_np(pthread_self(), sizeof(*info->affinity), info->affinity) < 0) 851 ublk_err("ublk dev %u thread %u set affinity failed", 852 info->dev->dev_info.dev_id, info->idx); 853 } 854 855 static __attribute__((noinline)) int __ublk_io_handler_fn(struct ublk_thread_info *info) 856 { 857 struct ublk_thread t = { 858 .dev = info->dev, 859 .idx = info->idx, 860 }; 861 int dev_id = info->dev->dev_info.dev_id; 862 int ret; 863 864 ret = ublk_thread_init(&t, info->extra_flags); 865 if (ret) { 866 ublk_err("ublk dev %d thread %u init failed\n", 867 dev_id, t.idx); 868 return ret; 869 } 870 sem_post(info->ready); 871 872 ublk_dbg(UBLK_DBG_THREAD, "tid %d: ublk dev %d thread %u started\n", 873 gettid(), dev_id, t.idx); 874 875 /* submit all io commands to ublk driver */ 876 ublk_submit_fetch_commands(&t); 877 do { 878 if (ublk_process_io(&t) < 0) 879 break; 880 } while (1); 881 882 ublk_dbg(UBLK_DBG_THREAD, "tid %d: ublk dev %d thread %d exiting\n", 883 gettid(), dev_id, t.idx); 884 ublk_thread_deinit(&t); 885 return 0; 886 } 887 888 static void *ublk_io_handler_fn(void *data) 889 { 890 struct ublk_thread_info *info = data; 891 892 /* 893 * IO perf is sensitive with queue pthread affinity on NUMA machine 894 * 895 * Set sched_affinity at beginning, so following allocated memory/pages 896 * could be CPU/NUMA aware. 897 */ 898 if (info->affinity) 899 ublk_thread_set_sched_affinity(info); 900 901 __ublk_io_handler_fn(info); 902 903 return NULL; 904 } 905 906 static void ublk_set_parameters(struct ublk_dev *dev) 907 { 908 int ret; 909 910 ret = ublk_ctrl_set_params(dev, &dev->tgt.params); 911 if (ret) 912 ublk_err("dev %d set basic parameter failed %d\n", 913 dev->dev_info.dev_id, ret); 914 } 915 916 static int ublk_send_dev_event(const struct dev_ctx *ctx, struct ublk_dev *dev, int dev_id) 917 { 918 uint64_t id; 919 int evtfd = ctx->_evtfd; 920 921 if (evtfd < 0) 922 return -EBADF; 923 924 if (dev_id >= 0) 925 id = dev_id + 1; 926 else 927 id = ERROR_EVTFD_DEVID; 928 929 if (dev && ctx->shadow_dev) 930 memcpy(&ctx->shadow_dev->q, &dev->q, sizeof(dev->q)); 931 932 if (write(evtfd, &id, sizeof(id)) != sizeof(id)) 933 return -EINVAL; 934 935 close(evtfd); 936 shmdt(ctx->shadow_dev); 937 938 return 0; 939 } 940 941 942 static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) 943 { 944 const struct ublksrv_ctrl_dev_info *dinfo = &dev->dev_info; 945 struct ublk_thread_info *tinfo; 946 unsigned long long extra_flags = 0; 947 cpu_set_t *affinity_buf; 948 void *thread_ret; 949 sem_t ready; 950 int ret, i; 951 952 ublk_dbg(UBLK_DBG_DEV, "%s enter\n", __func__); 953 954 tinfo = calloc(sizeof(struct ublk_thread_info), dev->nthreads); 955 if (!tinfo) 956 return -ENOMEM; 957 958 sem_init(&ready, 0, 0); 959 ret = ublk_dev_prep(ctx, dev); 960 if (ret) 961 return ret; 962 963 ret = ublk_ctrl_get_affinity(dev, &affinity_buf); 964 if (ret) 965 return ret; 966 967 if (ctx->auto_zc_fallback) 968 extra_flags = UBLKS_Q_AUTO_BUF_REG_FALLBACK; 969 if (ctx->no_ublk_fixed_fd) 970 extra_flags |= UBLKS_Q_NO_UBLK_FIXED_FD; 971 972 for (i = 0; i < dinfo->nr_hw_queues; i++) { 973 dev->q[i].dev = dev; 974 dev->q[i].q_id = i; 975 976 ret = ublk_queue_init(&dev->q[i], extra_flags); 977 if (ret) { 978 ublk_err("ublk dev %d queue %d init queue failed\n", 979 dinfo->dev_id, i); 980 goto fail; 981 } 982 } 983 984 for (i = 0; i < dev->nthreads; i++) { 985 tinfo[i].dev = dev; 986 tinfo[i].idx = i; 987 tinfo[i].ready = &ready; 988 tinfo[i].extra_flags = extra_flags; 989 990 /* 991 * If threads are not tied 1:1 to queues, setting thread 992 * affinity based on queue affinity makes little sense. 993 * However, thread CPU affinity has significant impact 994 * on performance, so to compare fairly, we'll still set 995 * thread CPU affinity based on queue affinity where 996 * possible. 997 */ 998 if (dev->nthreads == dinfo->nr_hw_queues) 999 tinfo[i].affinity = &affinity_buf[i]; 1000 pthread_create(&tinfo[i].thread, NULL, 1001 ublk_io_handler_fn, 1002 &tinfo[i]); 1003 } 1004 1005 for (i = 0; i < dev->nthreads; i++) 1006 sem_wait(&ready); 1007 free(affinity_buf); 1008 1009 /* everything is fine now, start us */ 1010 if (ctx->recovery) 1011 ret = ublk_ctrl_end_user_recovery(dev, getpid()); 1012 else { 1013 ublk_set_parameters(dev); 1014 ret = ublk_ctrl_start_dev(dev, getpid()); 1015 } 1016 if (ret < 0) { 1017 ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret); 1018 goto fail; 1019 } 1020 1021 ublk_ctrl_get_info(dev); 1022 if (ctx->fg) 1023 ublk_ctrl_dump(dev); 1024 else 1025 ublk_send_dev_event(ctx, dev, dev->dev_info.dev_id); 1026 1027 /* wait until we are terminated */ 1028 for (i = 0; i < dev->nthreads; i++) 1029 pthread_join(tinfo[i].thread, &thread_ret); 1030 free(tinfo); 1031 fail: 1032 for (i = 0; i < dinfo->nr_hw_queues; i++) 1033 ublk_queue_deinit(&dev->q[i]); 1034 ublk_dev_unprep(dev); 1035 ublk_dbg(UBLK_DBG_DEV, "%s exit\n", __func__); 1036 1037 return ret; 1038 } 1039 1040 static int wait_ublk_dev(const char *path, int evt_mask, unsigned timeout) 1041 { 1042 #define EV_SIZE (sizeof(struct inotify_event)) 1043 #define EV_BUF_LEN (128 * (EV_SIZE + 16)) 1044 struct pollfd pfd; 1045 int fd, wd; 1046 int ret = -EINVAL; 1047 const char *dev_name = basename(path); 1048 1049 fd = inotify_init(); 1050 if (fd < 0) { 1051 ublk_dbg(UBLK_DBG_DEV, "%s: inotify init failed\n", __func__); 1052 return fd; 1053 } 1054 1055 wd = inotify_add_watch(fd, "/dev", evt_mask); 1056 if (wd == -1) { 1057 ublk_dbg(UBLK_DBG_DEV, "%s: add watch for /dev failed\n", __func__); 1058 goto fail; 1059 } 1060 1061 pfd.fd = fd; 1062 pfd.events = POLL_IN; 1063 while (1) { 1064 int i = 0; 1065 char buffer[EV_BUF_LEN]; 1066 ret = poll(&pfd, 1, 1000 * timeout); 1067 1068 if (ret == -1) { 1069 ublk_err("%s: poll inotify failed: %d\n", __func__, ret); 1070 goto rm_watch; 1071 } else if (ret == 0) { 1072 ublk_err("%s: poll inotify timeout\n", __func__); 1073 ret = -ETIMEDOUT; 1074 goto rm_watch; 1075 } 1076 1077 ret = read(fd, buffer, EV_BUF_LEN); 1078 if (ret < 0) { 1079 ublk_err("%s: read inotify fd failed\n", __func__); 1080 goto rm_watch; 1081 } 1082 1083 while (i < ret) { 1084 struct inotify_event *event = (struct inotify_event *)&buffer[i]; 1085 1086 ublk_dbg(UBLK_DBG_DEV, "%s: inotify event %x %s\n", 1087 __func__, event->mask, event->name); 1088 if (event->mask & evt_mask) { 1089 if (!strcmp(event->name, dev_name)) { 1090 ret = 0; 1091 goto rm_watch; 1092 } 1093 } 1094 i += EV_SIZE + event->len; 1095 } 1096 } 1097 rm_watch: 1098 inotify_rm_watch(fd, wd); 1099 fail: 1100 close(fd); 1101 return ret; 1102 } 1103 1104 static int ublk_stop_io_daemon(const struct ublk_dev *dev) 1105 { 1106 int daemon_pid = dev->dev_info.ublksrv_pid; 1107 int dev_id = dev->dev_info.dev_id; 1108 char ublkc[64]; 1109 int ret = 0; 1110 1111 if (daemon_pid < 0) 1112 return 0; 1113 1114 /* daemon may be dead already */ 1115 if (kill(daemon_pid, 0) < 0) 1116 goto wait; 1117 1118 snprintf(ublkc, sizeof(ublkc), "/dev/%s%d", "ublkc", dev_id); 1119 1120 /* ublk char device may be gone already */ 1121 if (access(ublkc, F_OK) != 0) 1122 goto wait; 1123 1124 /* Wait until ublk char device is closed, when the daemon is shutdown */ 1125 ret = wait_ublk_dev(ublkc, IN_CLOSE, 10); 1126 /* double check and since it may be closed before starting inotify */ 1127 if (ret == -ETIMEDOUT) 1128 ret = kill(daemon_pid, 0) < 0; 1129 wait: 1130 waitpid(daemon_pid, NULL, 0); 1131 ublk_dbg(UBLK_DBG_DEV, "%s: pid %d dev_id %d ret %d\n", 1132 __func__, daemon_pid, dev_id, ret); 1133 1134 return ret; 1135 } 1136 1137 static int __cmd_dev_add(const struct dev_ctx *ctx) 1138 { 1139 unsigned nthreads = ctx->nthreads; 1140 unsigned nr_queues = ctx->nr_hw_queues; 1141 const char *tgt_type = ctx->tgt_type; 1142 unsigned depth = ctx->queue_depth; 1143 __u64 features; 1144 const struct ublk_tgt_ops *ops; 1145 struct ublksrv_ctrl_dev_info *info; 1146 struct ublk_dev *dev = NULL; 1147 int dev_id = ctx->dev_id; 1148 int ret, i; 1149 1150 ops = ublk_find_tgt(tgt_type); 1151 if (!ops) { 1152 ublk_err("%s: no such tgt type, type %s\n", 1153 __func__, tgt_type); 1154 ret = -ENODEV; 1155 goto fail; 1156 } 1157 1158 if (nr_queues > UBLK_MAX_QUEUES || depth > UBLK_QUEUE_DEPTH) { 1159 ublk_err("%s: invalid nr_queues or depth queues %u depth %u\n", 1160 __func__, nr_queues, depth); 1161 ret = -EINVAL; 1162 goto fail; 1163 } 1164 1165 /* default to 1:1 threads:queues if nthreads is unspecified */ 1166 if (!nthreads) 1167 nthreads = nr_queues; 1168 1169 if (nthreads > UBLK_MAX_THREADS) { 1170 ublk_err("%s: %u is too many threads (max %u)\n", 1171 __func__, nthreads, UBLK_MAX_THREADS); 1172 ret = -EINVAL; 1173 goto fail; 1174 } 1175 1176 if (nthreads != nr_queues && !ctx->per_io_tasks) { 1177 ublk_err("%s: threads %u must be same as queues %u if " 1178 "not using per_io_tasks\n", 1179 __func__, nthreads, nr_queues); 1180 ret = -EINVAL; 1181 goto fail; 1182 } 1183 1184 dev = ublk_ctrl_init(); 1185 if (!dev) { 1186 ublk_err("%s: can't alloc dev id %d, type %s\n", 1187 __func__, dev_id, tgt_type); 1188 ret = -ENOMEM; 1189 goto fail; 1190 } 1191 1192 /* kernel doesn't support get_features */ 1193 ret = ublk_ctrl_get_features(dev, &features); 1194 if (ret < 0) { 1195 ret = -EINVAL; 1196 goto fail; 1197 } 1198 1199 if (!(features & UBLK_F_CMD_IOCTL_ENCODE)) { 1200 ret = -ENOTSUP; 1201 goto fail; 1202 } 1203 1204 info = &dev->dev_info; 1205 info->dev_id = ctx->dev_id; 1206 info->nr_hw_queues = nr_queues; 1207 info->queue_depth = depth; 1208 info->flags = ctx->flags; 1209 if ((features & UBLK_F_QUIESCE) && 1210 (info->flags & UBLK_F_USER_RECOVERY)) 1211 info->flags |= UBLK_F_QUIESCE; 1212 dev->nthreads = nthreads; 1213 dev->per_io_tasks = ctx->per_io_tasks; 1214 dev->tgt.ops = ops; 1215 dev->tgt.sq_depth = depth; 1216 dev->tgt.cq_depth = depth; 1217 1218 for (i = 0; i < MAX_BACK_FILES; i++) { 1219 if (ctx->files[i]) { 1220 strcpy(dev->tgt.backing_file[i], ctx->files[i]); 1221 dev->tgt.nr_backing_files++; 1222 } 1223 } 1224 1225 if (ctx->recovery) 1226 ret = ublk_ctrl_start_user_recovery(dev); 1227 else 1228 ret = ublk_ctrl_add_dev(dev); 1229 if (ret < 0) { 1230 ublk_err("%s: can't add dev id %d, type %s ret %d\n", 1231 __func__, dev_id, tgt_type, ret); 1232 goto fail; 1233 } 1234 1235 ret = ublk_start_daemon(ctx, dev); 1236 ublk_dbg(UBLK_DBG_DEV, "%s: daemon exit %d\b", ret); 1237 if (ret < 0) 1238 ublk_ctrl_del_dev(dev); 1239 1240 fail: 1241 if (ret < 0) 1242 ublk_send_dev_event(ctx, dev, -1); 1243 if (dev) 1244 ublk_ctrl_deinit(dev); 1245 return ret; 1246 } 1247 1248 static int __cmd_dev_list(struct dev_ctx *ctx); 1249 1250 static int cmd_dev_add(struct dev_ctx *ctx) 1251 { 1252 int res; 1253 1254 if (ctx->fg) 1255 goto run; 1256 1257 ctx->_shmid = shmget(IPC_PRIVATE, sizeof(struct ublk_dev), IPC_CREAT | 0666); 1258 if (ctx->_shmid < 0) { 1259 ublk_err("%s: failed to shmget %s\n", __func__, strerror(errno)); 1260 exit(-1); 1261 } 1262 ctx->shadow_dev = (struct ublk_dev *)shmat(ctx->_shmid, NULL, 0); 1263 if (ctx->shadow_dev == (struct ublk_dev *)-1) { 1264 ublk_err("%s: failed to shmat %s\n", __func__, strerror(errno)); 1265 exit(-1); 1266 } 1267 ctx->_evtfd = eventfd(0, 0); 1268 if (ctx->_evtfd < 0) { 1269 ublk_err("%s: failed to create eventfd %s\n", __func__, strerror(errno)); 1270 exit(-1); 1271 } 1272 1273 res = fork(); 1274 if (res == 0) { 1275 int res2; 1276 1277 setsid(); 1278 res2 = fork(); 1279 if (res2 == 0) { 1280 /* prepare for detaching */ 1281 close(STDIN_FILENO); 1282 close(STDOUT_FILENO); 1283 close(STDERR_FILENO); 1284 run: 1285 res = __cmd_dev_add(ctx); 1286 return res; 1287 } else { 1288 /* detached from the foreground task */ 1289 exit(EXIT_SUCCESS); 1290 } 1291 } else if (res > 0) { 1292 uint64_t id; 1293 int exit_code = EXIT_FAILURE; 1294 1295 res = read(ctx->_evtfd, &id, sizeof(id)); 1296 close(ctx->_evtfd); 1297 if (res == sizeof(id) && id != ERROR_EVTFD_DEVID) { 1298 ctx->dev_id = id - 1; 1299 if (__cmd_dev_list(ctx) >= 0) 1300 exit_code = EXIT_SUCCESS; 1301 } 1302 shmdt(ctx->shadow_dev); 1303 shmctl(ctx->_shmid, IPC_RMID, NULL); 1304 /* wait for child and detach from it */ 1305 wait(NULL); 1306 if (exit_code == EXIT_FAILURE) 1307 ublk_err("%s: command failed\n", __func__); 1308 exit(exit_code); 1309 } else { 1310 exit(EXIT_FAILURE); 1311 } 1312 } 1313 1314 static int __cmd_dev_del(struct dev_ctx *ctx) 1315 { 1316 int number = ctx->dev_id; 1317 struct ublk_dev *dev; 1318 int ret; 1319 1320 dev = ublk_ctrl_init(); 1321 dev->dev_info.dev_id = number; 1322 1323 ret = ublk_ctrl_get_info(dev); 1324 if (ret < 0) 1325 goto fail; 1326 1327 ret = ublk_ctrl_stop_dev(dev); 1328 if (ret < 0) 1329 ublk_err("%s: stop dev %d failed ret %d\n", __func__, number, ret); 1330 1331 ret = ublk_stop_io_daemon(dev); 1332 if (ret < 0) 1333 ublk_err("%s: stop daemon id %d dev %d, ret %d\n", 1334 __func__, dev->dev_info.ublksrv_pid, number, ret); 1335 ublk_ctrl_del_dev(dev); 1336 fail: 1337 ublk_ctrl_deinit(dev); 1338 1339 return (ret >= 0) ? 0 : ret; 1340 } 1341 1342 static int cmd_dev_del(struct dev_ctx *ctx) 1343 { 1344 int i; 1345 1346 if (ctx->dev_id >= 0 || !ctx->all) 1347 return __cmd_dev_del(ctx); 1348 1349 for (i = 0; i < 255; i++) { 1350 ctx->dev_id = i; 1351 __cmd_dev_del(ctx); 1352 } 1353 return 0; 1354 } 1355 1356 static int __cmd_dev_list(struct dev_ctx *ctx) 1357 { 1358 struct ublk_dev *dev = ublk_ctrl_init(); 1359 int ret; 1360 1361 if (!dev) 1362 return -ENODEV; 1363 1364 dev->dev_info.dev_id = ctx->dev_id; 1365 1366 ret = ublk_ctrl_get_info(dev); 1367 if (ret < 0) { 1368 if (ctx->logging) 1369 ublk_err("%s: can't get dev info from %d: %d\n", 1370 __func__, ctx->dev_id, ret); 1371 } else { 1372 if (ctx->shadow_dev) 1373 memcpy(&dev->q, ctx->shadow_dev->q, sizeof(dev->q)); 1374 1375 ublk_ctrl_dump(dev); 1376 } 1377 1378 ublk_ctrl_deinit(dev); 1379 1380 return ret; 1381 } 1382 1383 static int cmd_dev_list(struct dev_ctx *ctx) 1384 { 1385 int i; 1386 1387 if (ctx->dev_id >= 0 || !ctx->all) 1388 return __cmd_dev_list(ctx); 1389 1390 ctx->logging = false; 1391 for (i = 0; i < 255; i++) { 1392 ctx->dev_id = i; 1393 __cmd_dev_list(ctx); 1394 } 1395 return 0; 1396 } 1397 1398 static int cmd_dev_get_features(void) 1399 { 1400 #define const_ilog2(x) (63 - __builtin_clzll(x)) 1401 #define FEAT_NAME(f) [const_ilog2(f)] = #f 1402 static const char *feat_map[] = { 1403 FEAT_NAME(UBLK_F_SUPPORT_ZERO_COPY), 1404 FEAT_NAME(UBLK_F_URING_CMD_COMP_IN_TASK), 1405 FEAT_NAME(UBLK_F_NEED_GET_DATA), 1406 FEAT_NAME(UBLK_F_USER_RECOVERY), 1407 FEAT_NAME(UBLK_F_USER_RECOVERY_REISSUE), 1408 FEAT_NAME(UBLK_F_UNPRIVILEGED_DEV), 1409 FEAT_NAME(UBLK_F_CMD_IOCTL_ENCODE), 1410 FEAT_NAME(UBLK_F_USER_COPY), 1411 FEAT_NAME(UBLK_F_ZONED), 1412 FEAT_NAME(UBLK_F_USER_RECOVERY_FAIL_IO), 1413 FEAT_NAME(UBLK_F_UPDATE_SIZE), 1414 FEAT_NAME(UBLK_F_AUTO_BUF_REG), 1415 FEAT_NAME(UBLK_F_QUIESCE), 1416 FEAT_NAME(UBLK_F_PER_IO_DAEMON), 1417 FEAT_NAME(UBLK_F_BUF_REG_OFF_DAEMON), 1418 }; 1419 struct ublk_dev *dev; 1420 __u64 features = 0; 1421 int ret; 1422 1423 dev = ublk_ctrl_init(); 1424 if (!dev) { 1425 fprintf(stderr, "ublksrv_ctrl_init failed id\n"); 1426 return -EOPNOTSUPP; 1427 } 1428 1429 ret = ublk_ctrl_get_features(dev, &features); 1430 if (!ret) { 1431 int i; 1432 1433 printf("ublk_drv features: 0x%llx\n", features); 1434 1435 for (i = 0; i < sizeof(features) * 8; i++) { 1436 const char *feat; 1437 1438 if (!((1ULL << i) & features)) 1439 continue; 1440 if (i < ARRAY_SIZE(feat_map)) 1441 feat = feat_map[i]; 1442 else 1443 feat = "unknown"; 1444 printf("0x%-16llx: %s\n", 1ULL << i, feat); 1445 } 1446 } 1447 1448 return ret; 1449 } 1450 1451 static int cmd_dev_update_size(struct dev_ctx *ctx) 1452 { 1453 struct ublk_dev *dev = ublk_ctrl_init(); 1454 struct ublk_params p; 1455 int ret = -EINVAL; 1456 1457 if (!dev) 1458 return -ENODEV; 1459 1460 if (ctx->dev_id < 0) { 1461 fprintf(stderr, "device id isn't provided\n"); 1462 goto out; 1463 } 1464 1465 dev->dev_info.dev_id = ctx->dev_id; 1466 ret = ublk_ctrl_get_params(dev, &p); 1467 if (ret < 0) { 1468 ublk_err("failed to get params %d %s\n", ret, strerror(-ret)); 1469 goto out; 1470 } 1471 1472 if (ctx->size & ((1 << p.basic.logical_bs_shift) - 1)) { 1473 ublk_err("size isn't aligned with logical block size\n"); 1474 ret = -EINVAL; 1475 goto out; 1476 } 1477 1478 ret = ublk_ctrl_update_size(dev, ctx->size >> 9); 1479 out: 1480 ublk_ctrl_deinit(dev); 1481 return ret; 1482 } 1483 1484 static int cmd_dev_quiesce(struct dev_ctx *ctx) 1485 { 1486 struct ublk_dev *dev = ublk_ctrl_init(); 1487 int ret = -EINVAL; 1488 1489 if (!dev) 1490 return -ENODEV; 1491 1492 if (ctx->dev_id < 0) { 1493 fprintf(stderr, "device id isn't provided for quiesce\n"); 1494 goto out; 1495 } 1496 dev->dev_info.dev_id = ctx->dev_id; 1497 ret = ublk_ctrl_quiesce_dev(dev, 10000); 1498 1499 out: 1500 ublk_ctrl_deinit(dev); 1501 return ret; 1502 } 1503 1504 static void __cmd_create_help(char *exe, bool recovery) 1505 { 1506 int i; 1507 1508 printf("%s %s -t [null|loop|stripe|fault_inject] [-q nr_queues] [-d depth] [-n dev_id]\n", 1509 exe, recovery ? "recover" : "add"); 1510 printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--auto_zc_fallback] [--debug_mask mask] [-r 0|1 ] [-g]\n"); 1511 printf("\t[-e 0|1 ] [-i 0|1] [--no_ublk_fixed_fd]\n"); 1512 printf("\t[--nthreads threads] [--per_io_tasks]\n"); 1513 printf("\t[target options] [backfile1] [backfile2] ...\n"); 1514 printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n"); 1515 printf("\tdefault: nthreads=nr_queues"); 1516 1517 for (i = 0; i < ARRAY_SIZE(tgt_ops_list); i++) { 1518 const struct ublk_tgt_ops *ops = tgt_ops_list[i]; 1519 1520 if (ops->usage) 1521 ops->usage(ops); 1522 } 1523 } 1524 1525 static void cmd_add_help(char *exe) 1526 { 1527 __cmd_create_help(exe, false); 1528 printf("\n"); 1529 } 1530 1531 static void cmd_recover_help(char *exe) 1532 { 1533 __cmd_create_help(exe, true); 1534 printf("\tPlease provide exact command line for creating this device with real dev_id\n"); 1535 printf("\n"); 1536 } 1537 1538 static int cmd_dev_help(char *exe) 1539 { 1540 cmd_add_help(exe); 1541 cmd_recover_help(exe); 1542 1543 printf("%s del [-n dev_id] -a \n", exe); 1544 printf("\t -a delete all devices -n delete specified device\n\n"); 1545 printf("%s list [-n dev_id] -a \n", exe); 1546 printf("\t -a list all devices, -n list specified device, default -a \n\n"); 1547 printf("%s features\n", exe); 1548 printf("%s update_size -n dev_id -s|--size size_in_bytes \n", exe); 1549 printf("%s quiesce -n dev_id\n", exe); 1550 return 0; 1551 } 1552 1553 int main(int argc, char *argv[]) 1554 { 1555 static const struct option longopts[] = { 1556 { "all", 0, NULL, 'a' }, 1557 { "type", 1, NULL, 't' }, 1558 { "number", 1, NULL, 'n' }, 1559 { "queues", 1, NULL, 'q' }, 1560 { "depth", 1, NULL, 'd' }, 1561 { "debug_mask", 1, NULL, 0 }, 1562 { "quiet", 0, NULL, 0 }, 1563 { "zero_copy", 0, NULL, 'z' }, 1564 { "foreground", 0, NULL, 0 }, 1565 { "recovery", 1, NULL, 'r' }, 1566 { "recovery_fail_io", 1, NULL, 'e'}, 1567 { "recovery_reissue", 1, NULL, 'i'}, 1568 { "get_data", 1, NULL, 'g'}, 1569 { "auto_zc", 0, NULL, 0 }, 1570 { "auto_zc_fallback", 0, NULL, 0 }, 1571 { "size", 1, NULL, 's'}, 1572 { "nthreads", 1, NULL, 0 }, 1573 { "per_io_tasks", 0, NULL, 0 }, 1574 { "no_ublk_fixed_fd", 0, NULL, 0 }, 1575 { 0, 0, 0, 0 } 1576 }; 1577 const struct ublk_tgt_ops *ops = NULL; 1578 int option_idx, opt; 1579 const char *cmd = argv[1]; 1580 struct dev_ctx ctx = { 1581 .queue_depth = 128, 1582 .nr_hw_queues = 2, 1583 .dev_id = -1, 1584 .tgt_type = "unknown", 1585 }; 1586 int ret = -EINVAL, i; 1587 int tgt_argc = 1; 1588 char *tgt_argv[MAX_NR_TGT_ARG] = { NULL }; 1589 int value; 1590 1591 if (argc == 1) 1592 return ret; 1593 1594 opterr = 0; 1595 optind = 2; 1596 while ((opt = getopt_long(argc, argv, "t:n:d:q:r:e:i:s:gaz", 1597 longopts, &option_idx)) != -1) { 1598 switch (opt) { 1599 case 'a': 1600 ctx.all = 1; 1601 break; 1602 case 'n': 1603 ctx.dev_id = strtol(optarg, NULL, 10); 1604 break; 1605 case 't': 1606 if (strlen(optarg) < sizeof(ctx.tgt_type)) 1607 strcpy(ctx.tgt_type, optarg); 1608 break; 1609 case 'q': 1610 ctx.nr_hw_queues = strtol(optarg, NULL, 10); 1611 break; 1612 case 'd': 1613 ctx.queue_depth = strtol(optarg, NULL, 10); 1614 break; 1615 case 'z': 1616 ctx.flags |= UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_USER_COPY; 1617 break; 1618 case 'r': 1619 value = strtol(optarg, NULL, 10); 1620 if (value) 1621 ctx.flags |= UBLK_F_USER_RECOVERY; 1622 break; 1623 case 'e': 1624 value = strtol(optarg, NULL, 10); 1625 if (value) 1626 ctx.flags |= UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_FAIL_IO; 1627 break; 1628 case 'i': 1629 value = strtol(optarg, NULL, 10); 1630 if (value) 1631 ctx.flags |= UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_REISSUE; 1632 break; 1633 case 'g': 1634 ctx.flags |= UBLK_F_NEED_GET_DATA; 1635 break; 1636 case 's': 1637 ctx.size = strtoull(optarg, NULL, 10); 1638 break; 1639 case 0: 1640 if (!strcmp(longopts[option_idx].name, "debug_mask")) 1641 ublk_dbg_mask = strtol(optarg, NULL, 16); 1642 if (!strcmp(longopts[option_idx].name, "quiet")) 1643 ublk_dbg_mask = 0; 1644 if (!strcmp(longopts[option_idx].name, "foreground")) 1645 ctx.fg = 1; 1646 if (!strcmp(longopts[option_idx].name, "auto_zc")) 1647 ctx.flags |= UBLK_F_AUTO_BUF_REG; 1648 if (!strcmp(longopts[option_idx].name, "auto_zc_fallback")) 1649 ctx.auto_zc_fallback = 1; 1650 if (!strcmp(longopts[option_idx].name, "nthreads")) 1651 ctx.nthreads = strtol(optarg, NULL, 10); 1652 if (!strcmp(longopts[option_idx].name, "per_io_tasks")) 1653 ctx.per_io_tasks = 1; 1654 if (!strcmp(longopts[option_idx].name, "no_ublk_fixed_fd")) 1655 ctx.no_ublk_fixed_fd = 1; 1656 break; 1657 case '?': 1658 /* 1659 * target requires every option must have argument 1660 */ 1661 if (argv[optind][0] == '-' || argv[optind - 1][0] != '-') { 1662 fprintf(stderr, "every target option requires argument: %s %s\n", 1663 argv[optind - 1], argv[optind]); 1664 exit(EXIT_FAILURE); 1665 } 1666 1667 if (tgt_argc < (MAX_NR_TGT_ARG - 1) / 2) { 1668 tgt_argv[tgt_argc++] = argv[optind - 1]; 1669 tgt_argv[tgt_argc++] = argv[optind]; 1670 } else { 1671 fprintf(stderr, "too many target options\n"); 1672 exit(EXIT_FAILURE); 1673 } 1674 optind += 1; 1675 break; 1676 } 1677 } 1678 1679 /* auto_zc_fallback depends on F_AUTO_BUF_REG & F_SUPPORT_ZERO_COPY */ 1680 if (ctx.auto_zc_fallback && 1681 !((ctx.flags & UBLK_F_AUTO_BUF_REG) && 1682 (ctx.flags & UBLK_F_SUPPORT_ZERO_COPY))) { 1683 ublk_err("%s: auto_zc_fallback is set but neither " 1684 "F_AUTO_BUF_REG nor F_SUPPORT_ZERO_COPY is enabled\n", 1685 __func__); 1686 return -EINVAL; 1687 } 1688 1689 i = optind; 1690 while (i < argc && ctx.nr_files < MAX_BACK_FILES) { 1691 ctx.files[ctx.nr_files++] = argv[i++]; 1692 } 1693 1694 ops = ublk_find_tgt(ctx.tgt_type); 1695 if (ops && ops->parse_cmd_line) { 1696 optind = 0; 1697 1698 tgt_argv[0] = ctx.tgt_type; 1699 ops->parse_cmd_line(&ctx, tgt_argc, tgt_argv); 1700 } 1701 1702 if (!strcmp(cmd, "add")) 1703 ret = cmd_dev_add(&ctx); 1704 else if (!strcmp(cmd, "recover")) { 1705 if (ctx.dev_id < 0) { 1706 fprintf(stderr, "device id isn't provided for recovering\n"); 1707 ret = -EINVAL; 1708 } else { 1709 ctx.recovery = 1; 1710 ret = cmd_dev_add(&ctx); 1711 } 1712 } else if (!strcmp(cmd, "del")) 1713 ret = cmd_dev_del(&ctx); 1714 else if (!strcmp(cmd, "list")) { 1715 ctx.all = 1; 1716 ret = cmd_dev_list(&ctx); 1717 } else if (!strcmp(cmd, "help")) 1718 ret = cmd_dev_help(argv[0]); 1719 else if (!strcmp(cmd, "features")) 1720 ret = cmd_dev_get_features(); 1721 else if (!strcmp(cmd, "update_size")) 1722 ret = cmd_dev_update_size(&ctx); 1723 else if (!strcmp(cmd, "quiesce")) 1724 ret = cmd_dev_quiesce(&ctx); 1725 else 1726 cmd_dev_help(argv[0]); 1727 1728 return ret; 1729 } 1730