1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef KUBLK_INTERNAL_H 3 #define KUBLK_INTERNAL_H 4 5 #include <unistd.h> 6 #include <stdlib.h> 7 #include <assert.h> 8 #include <stdio.h> 9 #include <stdarg.h> 10 #include <string.h> 11 #include <pthread.h> 12 #include <getopt.h> 13 #include <limits.h> 14 #include <poll.h> 15 #include <fcntl.h> 16 #include <sys/syscall.h> 17 #include <sys/mman.h> 18 #include <sys/ioctl.h> 19 #include <sys/inotify.h> 20 #include <sys/wait.h> 21 #include <sys/eventfd.h> 22 #include <sys/ipc.h> 23 #include <sys/shm.h> 24 #include <linux/io_uring.h> 25 #include <liburing.h> 26 #include <semaphore.h> 27 28 /* allow ublk_dep.h to override ublk_cmd.h */ 29 #include "ublk_dep.h" 30 #include <linux/ublk_cmd.h> 31 32 #include "utils.h" 33 34 #define MAX_BACK_FILES 4 35 36 /****************** part 1: libublk ********************/ 37 38 #define CTRL_DEV "/dev/ublk-control" 39 #define UBLKC_DEV "/dev/ublkc" 40 #define UBLKB_DEV "/dev/ublkb" 41 #define UBLK_CTRL_RING_DEPTH 32 42 #define ERROR_EVTFD_DEVID -2 43 44 #define UBLK_IO_MAX_BYTES (1 << 20) 45 #define UBLK_MAX_QUEUES_SHIFT 5 46 #define UBLK_MAX_QUEUES (1 << UBLK_MAX_QUEUES_SHIFT) 47 #define UBLK_MAX_THREADS_SHIFT 5 48 #define UBLK_MAX_THREADS (1 << UBLK_MAX_THREADS_SHIFT) 49 #define UBLK_QUEUE_DEPTH 1024 50 51 struct ublk_dev; 52 struct ublk_queue; 53 struct ublk_thread; 54 55 struct stripe_ctx { 56 /* stripe */ 57 unsigned int chunk_size; 58 }; 59 60 struct fault_inject_ctx { 61 /* fault_inject */ 62 unsigned long delay_us; 63 bool die_during_fetch; 64 }; 65 66 struct params_ctx { 67 __u32 types; 68 69 __u32 logical_bs_shift; 70 __u32 physical_bs_shift; 71 __u32 io_min_shift; 72 __u32 io_opt_shift; 73 __u32 max_sectors; 74 __u32 chunk_sectors; 75 __u64 dev_sectors; 76 77 __u32 max_open_zones; 78 __u32 max_active_zones; 79 __u32 max_zone_append_sectors; 80 }; 81 82 struct dev_ctx { 83 char tgt_type[16]; 84 unsigned long flags; 85 unsigned nr_hw_queues; 86 unsigned short nthreads; 87 unsigned queue_depth; 88 int dev_id; 89 int nr_files; 90 char *files[MAX_BACK_FILES]; 91 unsigned int logging:1; 92 unsigned int all:1; 93 unsigned int fg:1; 94 unsigned int recovery:1; 95 unsigned int auto_zc_fallback:1; 96 unsigned int per_io_tasks:1; 97 unsigned int no_ublk_fixed_fd:1; 98 unsigned int safe_stop:1; 99 unsigned int no_auto_part_scan:1; 100 unsigned int rdonly_shmem_buf:1; 101 unsigned int rotate_auto_buf:1; 102 __u32 integrity_flags; 103 __u8 metadata_size; 104 __u8 pi_offset; 105 __u8 csum_type; 106 __u8 tag_size; 107 __u16 io_desc_size; 108 109 int _evtfd; 110 int _shmid; 111 112 /* built from shmem, only for ublk_dump_dev() */ 113 struct ublk_dev *shadow_dev; 114 115 /* for 'update_size' command */ 116 unsigned long long size; 117 118 struct params_ctx params; 119 120 char *htlb_path; 121 122 union { 123 struct stripe_ctx stripe; 124 struct fault_inject_ctx fault_inject; 125 }; 126 }; 127 128 struct ublk_ctrl_cmd_data { 129 __u32 cmd_op; 130 #define CTRL_CMD_HAS_DATA 1 131 #define CTRL_CMD_HAS_BUF 2 132 __u32 flags; 133 134 __u64 data[2]; 135 __u64 addr; 136 __u32 len; 137 }; 138 139 struct ublk_io { 140 char *buf_addr; 141 void *integrity_buf; 142 143 #define UBLKS_IO_NEED_FETCH_RQ (1UL << 0) 144 #define UBLKS_IO_NEED_COMMIT_RQ_COMP (1UL << 1) 145 #define UBLKS_IO_FREE (1UL << 2) 146 #define UBLKS_IO_NEED_GET_DATA (1UL << 3) 147 #define UBLKS_IO_NEED_REG_BUF (1UL << 4) 148 unsigned short flags; 149 unsigned short refs; /* used by target code only */ 150 151 int tag; 152 153 int result; 154 155 unsigned short buf_index; 156 unsigned short tgt_ios; 157 unsigned char auto_buf_phase; 158 void *private_data; 159 }; 160 161 struct ublk_tgt_ops { 162 const char *name; 163 int (*init_tgt)(const struct dev_ctx *ctx, struct ublk_dev *); 164 void (*deinit_tgt)(struct ublk_dev *); 165 166 void (*pre_fetch_io)(struct ublk_thread *t, struct ublk_queue *q, 167 int tag, bool batch); 168 int (*queue_io)(struct ublk_thread *, struct ublk_queue *, int tag); 169 void (*tgt_io_done)(struct ublk_thread *, struct ublk_queue *, 170 const struct io_uring_cqe *); 171 172 /* 173 * Target specific command line handling 174 * 175 * each option requires argument for target command line 176 */ 177 void (*parse_cmd_line)(struct dev_ctx *ctx, int argc, char *argv[]); 178 void (*usage)(const struct ublk_tgt_ops *ops); 179 180 /* return buffer index for UBLK_F_AUTO_BUF_REG */ 181 unsigned short (*buf_index)(const struct ublk_thread *t, 182 const struct ublk_queue *, int tag); 183 }; 184 185 struct ublk_tgt { 186 unsigned long dev_size; 187 unsigned int sq_depth; 188 unsigned int cq_depth; 189 const struct ublk_tgt_ops *ops; 190 struct ublk_params params; 191 192 int nr_backing_files; 193 unsigned long backing_file_size[MAX_BACK_FILES]; 194 char backing_file[MAX_BACK_FILES][PATH_MAX]; 195 }; 196 197 struct ublk_queue { 198 int q_id; 199 int q_depth; 200 struct ublk_dev *dev; 201 const struct ublk_tgt_ops *tgt_ops; 202 struct ublksrv_io_desc *io_cmd_buf; 203 204 /* borrow three bit of ublk uapi flags, which may never be used */ 205 #define UBLKS_Q_AUTO_BUF_REG_FALLBACK (1ULL << 63) 206 #define UBLKS_Q_NO_UBLK_FIXED_FD (1ULL << 62) 207 #define UBLKS_Q_PREPARED (1ULL << 61) 208 #define UBLKS_Q_ROTATE_AUTO_BUF (1ULL << 60) 209 __u64 flags; 210 int ublk_fd; /* cached ublk char device fd */ 211 __u8 metadata_size; 212 __u16 io_desc_size; 213 struct ublk_io ios[UBLK_QUEUE_DEPTH]; 214 215 /* used for prep io commands */ 216 pthread_spinlock_t lock; 217 }; 218 219 /* align with `ublk_elem_header` */ 220 struct ublk_batch_elem { 221 __u16 tag; 222 __u16 buf_index; 223 __s32 result; 224 __u64 buf_addr; 225 }; 226 227 struct batch_commit_buf { 228 unsigned short q_id; 229 unsigned short buf_idx; 230 void *elem; 231 unsigned short done; 232 unsigned short count; 233 }; 234 235 struct batch_fetch_buf { 236 struct io_uring_buf_ring *br; 237 void *fetch_buf; 238 unsigned int fetch_buf_size; 239 unsigned int fetch_buf_off; 240 }; 241 242 struct ublk_thread { 243 /* Thread-local copy of queue-to-thread mapping for this thread */ 244 unsigned char q_map[UBLK_MAX_QUEUES]; 245 246 struct ublk_dev *dev; 247 unsigned short idx; 248 unsigned short nr_queues; 249 250 #define UBLKS_T_STOPPING (1U << 0) 251 #define UBLKS_T_IDLE (1U << 1) 252 #define UBLKS_T_BATCH_IO (1U << 31) /* readonly */ 253 unsigned state; 254 unsigned int cmd_inflight; 255 unsigned int io_inflight; 256 257 unsigned short nr_bufs; 258 unsigned short auto_buf_stride; 259 260 /* followings are for BATCH_IO */ 261 unsigned short commit_buf_start; 262 unsigned char commit_buf_elem_size; 263 /* 264 * We just support single device, so pre-calculate commit/prep flags 265 */ 266 unsigned short cmd_flags; 267 unsigned int nr_commit_buf; 268 unsigned int commit_buf_size; 269 void *commit_buf; 270 #define UBLKS_T_COMMIT_BUF_INV_IDX ((unsigned short)-1) 271 struct allocator commit_buf_alloc; 272 struct batch_commit_buf *commit; 273 /* FETCH_IO_CMDS buffer */ 274 unsigned short nr_fetch_bufs; 275 struct batch_fetch_buf *fetch; 276 277 struct io_uring ring; 278 }; 279 280 struct ublk_dev { 281 struct ublk_tgt tgt; 282 struct ublksrv_ctrl_dev_info dev_info; 283 struct ublk_queue q[UBLK_MAX_QUEUES]; 284 unsigned nthreads; 285 unsigned per_io_tasks; 286 287 int fds[MAX_BACK_FILES + 1]; /* fds[0] points to /dev/ublkcN */ 288 int nr_fds; 289 int ctrl_fd; 290 struct io_uring ring; 291 292 void *private_data; 293 }; 294 295 extern int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io); 296 297 static inline int ublk_setup_ring(struct io_uring *r, int depth, 298 int cq_depth, unsigned int flags) 299 { 300 struct io_uring_params p; 301 302 memset(&p, 0, sizeof(p)); 303 p.flags = flags | IORING_SETUP_CQSIZE; 304 p.cq_entries = cq_depth; 305 306 return io_uring_queue_init_params(depth, r, &p); 307 } 308 309 /* ctrl.c: control commands */ 310 struct ublk_dev *ublk_ctrl_init(void); 311 void ublk_ctrl_deinit(struct ublk_dev *dev); 312 int __ublk_ctrl_cmd(struct ublk_dev *dev, struct ublk_ctrl_cmd_data *data); 313 int ublk_ctrl_add_dev(struct ublk_dev *dev); 314 int ublk_ctrl_del_dev(struct ublk_dev *dev); 315 int ublk_ctrl_get_info(struct ublk_dev *dev); 316 int ublk_ctrl_set_params(struct ublk_dev *dev, struct ublk_params *params); 317 int ublk_ctrl_get_params(struct ublk_dev *dev, struct ublk_params *params); 318 int ublk_ctrl_get_features(struct ublk_dev *dev, __u64 *features); 319 int ublk_ctrl_start_dev(struct ublk_dev *dev, int daemon_pid); 320 int ublk_ctrl_stop_dev(struct ublk_dev *dev); 321 int ublk_ctrl_try_stop_dev(struct ublk_dev *dev); 322 int ublk_ctrl_start_user_recovery(struct ublk_dev *dev); 323 int ublk_ctrl_end_user_recovery(struct ublk_dev *dev, int daemon_pid); 324 int ublk_ctrl_update_size(struct ublk_dev *dev, __u64 nr_sects); 325 int ublk_ctrl_quiesce_dev(struct ublk_dev *dev, unsigned int timeout_ms); 326 int ublk_ctrl_reg_buf(struct ublk_dev *dev, void *addr, size_t size, 327 __u32 flags); 328 329 static inline int __ublk_use_batch_io(__u64 flags) 330 { 331 return flags & UBLK_F_BATCH_IO; 332 } 333 334 static inline int ublk_queue_batch_io(const struct ublk_queue *q) 335 { 336 return __ublk_use_batch_io(q->flags); 337 } 338 339 static inline int ublk_dev_batch_io(const struct ublk_dev *dev) 340 { 341 return __ublk_use_batch_io(dev->dev_info.flags); 342 } 343 344 /* only work for handle single device in this pthread context */ 345 static inline int ublk_thread_batch_io(const struct ublk_thread *t) 346 { 347 return t->state & UBLKS_T_BATCH_IO; 348 } 349 350 static inline void ublk_set_integrity_params(const struct dev_ctx *ctx, 351 struct ublk_params *params) 352 { 353 if (!ctx->metadata_size) 354 return; 355 356 params->types |= UBLK_PARAM_TYPE_INTEGRITY; 357 params->integrity = (struct ublk_param_integrity) { 358 .flags = ctx->integrity_flags, 359 .interval_exp = params->basic.logical_bs_shift, 360 .metadata_size = ctx->metadata_size, 361 .pi_offset = ctx->pi_offset, 362 .csum_type = ctx->csum_type, 363 .tag_size = ctx->tag_size, 364 }; 365 } 366 367 static inline size_t ublk_integrity_len(const struct ublk_queue *q, size_t len) 368 { 369 /* All targets currently use interval_exp = logical_bs_shift = 9 */ 370 return (len >> 9) * q->metadata_size; 371 } 372 373 static inline size_t 374 ublk_integrity_data_len(const struct ublk_queue *q, size_t integrity_len) 375 { 376 return (integrity_len / q->metadata_size) << 9; 377 } 378 379 static inline int ublk_io_auto_zc_fallback(const struct ublksrv_io_desc *iod) 380 { 381 return !!(iod->op_flags & UBLK_IO_F_NEED_REG_BUF); 382 } 383 384 static inline __u64 ublk_user_copy_offset(unsigned q_id, unsigned tag) 385 { 386 return UBLKSRV_IO_BUF_OFFSET + 387 ((__u64)q_id << UBLK_QID_OFF | (__u64)tag << UBLK_TAG_OFF); 388 } 389 390 static inline int is_target_io(__u64 user_data) 391 { 392 return (user_data & (1ULL << 63)) != 0; 393 } 394 395 static inline __u64 build_user_data(unsigned tag, unsigned op, 396 unsigned tgt_data, unsigned q_id, unsigned is_target_io) 397 { 398 /* we only have 7 bits to encode q_id */ 399 _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7, "UBLK_MAX_QUEUES_SHIFT must be <= 7"); 400 ublk_assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16) && !(q_id >> 7)); 401 402 return tag | ((__u64)op << 16) | ((__u64)tgt_data << 24) | 403 (__u64)q_id << 56 | (__u64)is_target_io << 63; 404 } 405 406 static inline unsigned int user_data_to_tag(__u64 user_data) 407 { 408 return user_data & 0xffff; 409 } 410 411 static inline unsigned int user_data_to_op(__u64 user_data) 412 { 413 return (user_data >> 16) & 0xff; 414 } 415 416 static inline unsigned int user_data_to_tgt_data(__u64 user_data) 417 { 418 return (user_data >> 24) & 0xffff; 419 } 420 421 static inline unsigned int user_data_to_q_id(__u64 user_data) 422 { 423 return (user_data >> 56) & 0x7f; 424 } 425 426 static inline unsigned short ublk_cmd_op_nr(unsigned int op) 427 { 428 return _IOC_NR(op); 429 } 430 431 static inline struct ublk_queue *ublk_io_to_queue(const struct ublk_io *io) 432 { 433 return container_of(io, struct ublk_queue, ios[io->tag]); 434 } 435 436 static inline int ublk_io_alloc_sqes(struct ublk_thread *t, 437 struct io_uring_sqe *sqes[], int nr_sqes) 438 { 439 struct io_uring *ring = &t->ring; 440 unsigned left = io_uring_sq_space_left(ring); 441 int i; 442 443 if (left < nr_sqes) 444 io_uring_submit(ring); 445 446 for (i = 0; i < nr_sqes; i++) { 447 sqes[i] = io_uring_get_sqe(ring); 448 if (!sqes[i]) 449 return i; 450 } 451 452 return nr_sqes; 453 } 454 455 static inline int ublk_get_registered_fd(struct ublk_queue *q, int fd_index) 456 { 457 if (q->flags & UBLKS_Q_NO_UBLK_FIXED_FD) { 458 if (fd_index == 0) 459 /* Return the raw ublk FD for index 0 */ 460 return q->ublk_fd; 461 /* Adjust index for backing files (index 1 becomes 0, etc.) */ 462 return fd_index - 1; 463 } 464 return fd_index; 465 } 466 467 static inline void __io_uring_prep_buf_reg_unreg(struct io_uring_sqe *sqe, 468 struct ublk_queue *q, int tag, int q_id, __u64 index) 469 { 470 struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd; 471 int dev_fd = ublk_get_registered_fd(q, 0); 472 473 io_uring_prep_read(sqe, dev_fd, 0, 0, 0); 474 sqe->opcode = IORING_OP_URING_CMD; 475 if (q->flags & UBLKS_Q_NO_UBLK_FIXED_FD) 476 sqe->flags &= ~IOSQE_FIXED_FILE; 477 else 478 sqe->flags |= IOSQE_FIXED_FILE; 479 480 cmd->tag = tag; 481 cmd->addr = index; 482 cmd->q_id = q_id; 483 } 484 485 static inline void io_uring_prep_buf_register(struct io_uring_sqe *sqe, 486 struct ublk_queue *q, int tag, int q_id, __u64 index) 487 { 488 __io_uring_prep_buf_reg_unreg(sqe, q, tag, q_id, index); 489 sqe->cmd_op = UBLK_U_IO_REGISTER_IO_BUF; 490 } 491 492 static inline void io_uring_prep_buf_unregister(struct io_uring_sqe *sqe, 493 struct ublk_queue *q, int tag, int q_id, __u64 index) 494 { 495 __io_uring_prep_buf_reg_unreg(sqe, q, tag, q_id, index); 496 sqe->cmd_op = UBLK_U_IO_UNREGISTER_IO_BUF; 497 } 498 499 static inline void *ublk_get_sqe_cmd(const struct io_uring_sqe *sqe) 500 { 501 return (void *)&sqe->cmd; 502 } 503 504 static inline void ublk_set_io_res(struct ublk_queue *q, int tag, int res) 505 { 506 q->ios[tag].result = res; 507 } 508 509 static inline int ublk_get_io_res(const struct ublk_queue *q, unsigned tag) 510 { 511 return q->ios[tag].result; 512 } 513 514 static inline void ublk_mark_io_done(struct ublk_io *io, int res) 515 { 516 io->flags |= (UBLKS_IO_NEED_COMMIT_RQ_COMP | UBLKS_IO_FREE); 517 io->result = res; 518 } 519 520 static inline const struct ublksrv_io_desc *ublk_get_iod(const struct ublk_queue *q, __u16 tag) 521 { 522 return (void *)q->io_cmd_buf + tag * (size_t)q->io_desc_size; 523 } 524 525 static inline void ublk_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op) 526 { 527 __u32 *addr = (__u32 *)&sqe->off; 528 529 addr[0] = cmd_op; 530 addr[1] = 0; 531 } 532 533 static inline unsigned short ublk_batch_io_buf_idx( 534 const struct ublk_thread *t, const struct ublk_queue *q, 535 unsigned tag); 536 537 static inline unsigned short ublk_io_buf_idx(const struct ublk_thread *t, 538 const struct ublk_queue *q, 539 unsigned tag) 540 { 541 if (ublk_queue_batch_io(q)) 542 return ublk_batch_io_buf_idx(t, q, tag); 543 return q->ios[tag].buf_index; 544 } 545 546 static inline struct ublk_io *ublk_get_io(struct ublk_queue *q, unsigned tag) 547 { 548 return &q->ios[tag]; 549 } 550 551 static inline int ublk_completed_tgt_io(struct ublk_thread *t, 552 struct ublk_queue *q, unsigned tag) 553 { 554 struct ublk_io *io = ublk_get_io(q, tag); 555 556 t->io_inflight--; 557 558 return --io->tgt_ios == 0; 559 } 560 561 static inline bool ublk_queue_use_zc(const struct ublk_queue *q) 562 { 563 return !!(q->flags & UBLK_F_SUPPORT_ZERO_COPY); 564 } 565 566 static inline bool ublk_queue_use_auto_zc(const struct ublk_queue *q) 567 { 568 return !!(q->flags & UBLK_F_AUTO_BUF_REG); 569 } 570 571 static inline bool ublk_queue_auto_zc_fallback(const struct ublk_queue *q) 572 { 573 return !!(q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK); 574 } 575 576 static inline bool ublk_queue_use_user_copy(const struct ublk_queue *q) 577 { 578 return !!(q->flags & UBLK_F_USER_COPY); 579 } 580 581 static inline int ublk_queue_no_buf(const struct ublk_queue *q) 582 { 583 return ublk_queue_use_zc(q) || ublk_queue_use_auto_zc(q); 584 } 585 586 static inline int ublk_batch_commit_prepared(struct batch_commit_buf *cb) 587 { 588 return cb->buf_idx != UBLKS_T_COMMIT_BUF_INV_IDX; 589 } 590 591 static inline unsigned ublk_queue_idx_in_thread(const struct ublk_thread *t, 592 const struct ublk_queue *q) 593 { 594 unsigned char idx; 595 596 idx = t->q_map[q->q_id]; 597 ublk_assert(idx != 0); 598 return idx - 1; 599 } 600 601 /* 602 * Each IO's buffer index has to be calculated by this helper for 603 * UBLKS_T_BATCH_IO 604 */ 605 static inline unsigned short ublk_batch_io_buf_idx( 606 const struct ublk_thread *t, const struct ublk_queue *q, 607 unsigned tag) 608 { 609 unsigned short base = ublk_queue_idx_in_thread(t, q) * q->q_depth + tag; 610 611 if (q->flags & UBLKS_Q_ROTATE_AUTO_BUF) 612 return base + q->ios[tag].auto_buf_phase * t->auto_buf_stride; 613 return base; 614 } 615 616 static inline unsigned short ublk_batch_io_buf_idx_next( 617 const struct ublk_thread *t, struct ublk_queue *q, 618 unsigned tag) 619 { 620 if (q->flags & UBLKS_Q_ROTATE_AUTO_BUF) 621 q->ios[tag].auto_buf_phase ^= 1; 622 return ublk_batch_io_buf_idx(t, q, tag); 623 } 624 625 /* Queue UBLK_U_IO_PREP_IO_CMDS for a specific queue with batch elements */ 626 int ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_queue *q); 627 /* Start fetching I/O commands using multishot UBLK_U_IO_FETCH_IO_CMDS */ 628 void ublk_batch_start_fetch(struct ublk_thread *t); 629 /* Handle completion of batch I/O commands (prep/commit) */ 630 void ublk_batch_compl_cmd(struct ublk_thread *t, 631 const struct io_uring_cqe *cqe); 632 /* Initialize batch I/O state and calculate buffer parameters */ 633 void ublk_batch_prepare(struct ublk_thread *t); 634 /* Allocate and register commit buffers for batch operations */ 635 int ublk_batch_alloc_buf(struct ublk_thread *t); 636 /* Free commit buffers and cleanup batch allocator */ 637 void ublk_batch_free_buf(struct ublk_thread *t); 638 639 /* Prepare a new commit buffer for batching completed I/O operations */ 640 void ublk_batch_prep_commit(struct ublk_thread *t); 641 /* Submit UBLK_U_IO_COMMIT_IO_CMDS with batched completed I/O operations */ 642 void ublk_batch_commit_io_cmds(struct ublk_thread *t); 643 /* Add a completed I/O operation to the current batch commit buffer */ 644 void ublk_batch_complete_io(struct ublk_thread *t, struct ublk_queue *q, 645 unsigned tag, int res); 646 void ublk_batch_setup_map(unsigned char (*q_thread_map)[UBLK_MAX_QUEUES], 647 int nthreads, int queues); 648 649 static inline int ublk_complete_io(struct ublk_thread *t, struct ublk_queue *q, 650 unsigned tag, int res) 651 { 652 if (ublk_queue_batch_io(q)) { 653 ublk_batch_complete_io(t, q, tag, res); 654 return 0; 655 } else { 656 struct ublk_io *io = &q->ios[tag]; 657 658 ublk_mark_io_done(io, res); 659 return ublk_queue_io_cmd(t, io); 660 } 661 } 662 663 static inline void ublk_queued_tgt_io(struct ublk_thread *t, struct ublk_queue *q, 664 unsigned tag, int queued) 665 { 666 if (queued < 0) 667 ublk_complete_io(t, q, tag, queued); 668 else { 669 struct ublk_io *io = ublk_get_io(q, tag); 670 671 t->io_inflight += queued; 672 io->tgt_ios = queued; 673 io->result = 0; 674 } 675 } 676 677 /* shared memory zero-copy support */ 678 #define UBLK_BUF_MAX 256 679 680 struct ublk_shmem_entry { 681 int fd; 682 void *mmap_base; 683 size_t size; 684 }; 685 686 extern struct ublk_shmem_entry shmem_table[UBLK_BUF_MAX]; 687 extern int shmem_count; 688 689 extern const struct ublk_tgt_ops null_tgt_ops; 690 extern const struct ublk_tgt_ops loop_tgt_ops; 691 extern const struct ublk_tgt_ops stripe_tgt_ops; 692 extern const struct ublk_tgt_ops fault_inject_tgt_ops; 693 694 void backing_file_tgt_deinit(struct ublk_dev *dev); 695 int backing_file_tgt_init(struct ublk_dev *dev, unsigned int nr_direct); 696 697 #endif 698