1 /*- 2 * Copyright (c) 1997-2007 Kenneth D. Merry 3 * Copyright (c) 2013, 2014, 2015 Spectra Logic Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions, and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * substantially similar to the "NO WARRANTY" disclaimer below 14 * ("Disclaimer") and any redistribution must be conditioned upon 15 * including a substantially similar Disclaimer requirement for further 16 * binary redistribution. 17 * 18 * NO WARRANTY 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 28 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGES. 30 * 31 * Authors: Ken Merry (Spectra Logic Corporation) 32 */ 33 34 /* 35 * This is eventually intended to be: 36 * - A basic data transfer/copy utility 37 * - A simple benchmark utility 38 * - An example of how to use the asynchronous pass(4) driver interface. 39 */ 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/ioctl.h> 44 #include <sys/stdint.h> 45 #include <sys/types.h> 46 #include <sys/endian.h> 47 #include <sys/param.h> 48 #include <sys/sbuf.h> 49 #include <sys/stat.h> 50 #include <sys/event.h> 51 #include <sys/time.h> 52 #include <sys/uio.h> 53 #include <vm/vm.h> 54 #include <machine/bus.h> 55 #include <sys/bus.h> 56 #include <sys/bus_dma.h> 57 #include <sys/mtio.h> 58 #include <sys/conf.h> 59 #include <sys/disk.h> 60 61 #include <stdio.h> 62 #include <stdlib.h> 63 #include <semaphore.h> 64 #include <string.h> 65 #include <unistd.h> 66 #include <inttypes.h> 67 #include <limits.h> 68 #include <fcntl.h> 69 #include <ctype.h> 70 #include <err.h> 71 #include <libutil.h> 72 #include <pthread.h> 73 #include <assert.h> 74 #include <bsdxml.h> 75 76 #include <cam/cam.h> 77 #include <cam/cam_debug.h> 78 #include <cam/cam_ccb.h> 79 #include <cam/scsi/scsi_all.h> 80 #include <cam/scsi/scsi_da.h> 81 #include <cam/scsi/scsi_pass.h> 82 #include <cam/scsi/scsi_message.h> 83 #include <cam/scsi/smp_all.h> 84 #include <camlib.h> 85 #include <mtlib.h> 86 #include <zlib.h> 87 88 typedef enum { 89 CAMDD_CMD_NONE = 0x00000000, 90 CAMDD_CMD_HELP = 0x00000001, 91 CAMDD_CMD_WRITE = 0x00000002, 92 CAMDD_CMD_READ = 0x00000003 93 } camdd_cmdmask; 94 95 typedef enum { 96 CAMDD_ARG_NONE = 0x00000000, 97 CAMDD_ARG_VERBOSE = 0x00000001, 98 CAMDD_ARG_DEVICE = 0x00000002, 99 CAMDD_ARG_BUS = 0x00000004, 100 CAMDD_ARG_TARGET = 0x00000008, 101 CAMDD_ARG_LUN = 0x00000010, 102 CAMDD_ARG_UNIT = 0x00000020, 103 CAMDD_ARG_TIMEOUT = 0x00000040, 104 CAMDD_ARG_ERR_RECOVER = 0x00000080, 105 CAMDD_ARG_RETRIES = 0x00000100 106 } camdd_argmask; 107 108 typedef enum { 109 CAMDD_DEV_NONE = 0x00, 110 CAMDD_DEV_PASS = 0x01, 111 CAMDD_DEV_FILE = 0x02 112 } camdd_dev_type; 113 114 struct camdd_io_opts { 115 camdd_dev_type dev_type; 116 char *dev_name; 117 uint64_t blocksize; 118 uint64_t queue_depth; 119 uint64_t offset; 120 int min_cmd_size; 121 int write_dev; 122 uint64_t debug; 123 }; 124 125 typedef enum { 126 CAMDD_BUF_NONE, 127 CAMDD_BUF_DATA, 128 CAMDD_BUF_INDIRECT 129 } camdd_buf_type; 130 131 struct camdd_buf_indirect { 132 /* 133 * Pointer to the source buffer. 134 */ 135 struct camdd_buf *src_buf; 136 137 /* 138 * Offset into the source buffer, in bytes. 139 */ 140 uint64_t offset; 141 /* 142 * Pointer to the starting point in the source buffer. 143 */ 144 uint8_t *start_ptr; 145 146 /* 147 * Length of this chunk in bytes. 148 */ 149 size_t len; 150 }; 151 152 struct camdd_buf_data { 153 /* 154 * Buffer allocated when we allocate this camdd_buf. This should 155 * be the size of the blocksize for this device. 156 */ 157 uint8_t *buf; 158 159 /* 160 * The amount of backing store allocated in buf. Generally this 161 * will be the blocksize of the device. 162 */ 163 uint32_t alloc_len; 164 165 /* 166 * The amount of data that was put into the buffer (on reads) or 167 * the amount of data we have put onto the src_list so far (on 168 * writes). 169 */ 170 uint32_t fill_len; 171 172 /* 173 * The amount of data that was not transferred. 174 */ 175 uint32_t resid; 176 177 /* 178 * Starting byte offset on the reader. 179 */ 180 uint64_t src_start_offset; 181 182 /* 183 * CCB used for pass(4) device targets. 184 */ 185 union ccb ccb; 186 187 /* 188 * Number of scatter/gather segments. 189 */ 190 int sg_count; 191 192 /* 193 * Set if we had to tack on an extra buffer to round the transfer 194 * up to a sector size. 195 */ 196 int extra_buf; 197 198 /* 199 * Scatter/gather list used generally when we're the writer for a 200 * pass(4) device. 201 */ 202 bus_dma_segment_t *segs; 203 204 /* 205 * Scatter/gather list used generally when we're the writer for a 206 * file or block device; 207 */ 208 struct iovec *iovec; 209 }; 210 211 union camdd_buf_types { 212 struct camdd_buf_indirect indirect; 213 struct camdd_buf_data data; 214 }; 215 216 typedef enum { 217 CAMDD_STATUS_NONE, 218 CAMDD_STATUS_OK, 219 CAMDD_STATUS_SHORT_IO, 220 CAMDD_STATUS_EOF, 221 CAMDD_STATUS_ERROR 222 } camdd_buf_status; 223 224 struct camdd_buf { 225 camdd_buf_type buf_type; 226 union camdd_buf_types buf_type_spec; 227 228 camdd_buf_status status; 229 230 uint64_t lba; 231 size_t len; 232 233 /* 234 * A reference count of how many indirect buffers point to this 235 * buffer. 236 */ 237 int refcount; 238 239 /* 240 * A link back to our parent device. 241 */ 242 struct camdd_dev *dev; 243 STAILQ_ENTRY(camdd_buf) links; 244 STAILQ_ENTRY(camdd_buf) work_links; 245 246 /* 247 * A count of the buffers on the src_list. 248 */ 249 int src_count; 250 251 /* 252 * List of buffers from our partner thread that are the components 253 * of this buffer for the I/O. Uses src_links. 254 */ 255 STAILQ_HEAD(,camdd_buf) src_list; 256 STAILQ_ENTRY(camdd_buf) src_links; 257 }; 258 259 #define NUM_DEV_TYPES 2 260 261 struct camdd_dev_pass { 262 int scsi_dev_type; 263 struct cam_device *dev; 264 uint64_t max_sector; 265 uint32_t block_len; 266 uint32_t cpi_maxio; 267 }; 268 269 typedef enum { 270 CAMDD_FILE_NONE, 271 CAMDD_FILE_REG, 272 CAMDD_FILE_STD, 273 CAMDD_FILE_PIPE, 274 CAMDD_FILE_DISK, 275 CAMDD_FILE_TAPE, 276 CAMDD_FILE_TTY, 277 CAMDD_FILE_MEM 278 } camdd_file_type; 279 280 typedef enum { 281 CAMDD_FF_NONE = 0x00, 282 CAMDD_FF_CAN_SEEK = 0x01 283 } camdd_file_flags; 284 285 struct camdd_dev_file { 286 int fd; 287 struct stat sb; 288 char filename[MAXPATHLEN + 1]; 289 camdd_file_type file_type; 290 camdd_file_flags file_flags; 291 uint8_t *tmp_buf; 292 }; 293 294 struct camdd_dev_block { 295 int fd; 296 uint64_t size_bytes; 297 uint32_t block_len; 298 }; 299 300 union camdd_dev_spec { 301 struct camdd_dev_pass pass; 302 struct camdd_dev_file file; 303 struct camdd_dev_block block; 304 }; 305 306 typedef enum { 307 CAMDD_DEV_FLAG_NONE = 0x00, 308 CAMDD_DEV_FLAG_EOF = 0x01, 309 CAMDD_DEV_FLAG_PEER_EOF = 0x02, 310 CAMDD_DEV_FLAG_ACTIVE = 0x04, 311 CAMDD_DEV_FLAG_EOF_SENT = 0x08, 312 CAMDD_DEV_FLAG_EOF_QUEUED = 0x10 313 } camdd_dev_flags; 314 315 struct camdd_dev { 316 camdd_dev_type dev_type; 317 union camdd_dev_spec dev_spec; 318 camdd_dev_flags flags; 319 char device_name[MAXPATHLEN+1]; 320 uint32_t blocksize; 321 uint32_t sector_size; 322 uint64_t max_sector; 323 uint64_t sector_io_limit; 324 int min_cmd_size; 325 int write_dev; 326 int retry_count; 327 int io_timeout; 328 int debug; 329 uint64_t start_offset_bytes; 330 uint64_t next_io_pos_bytes; 331 uint64_t next_peer_pos_bytes; 332 uint64_t next_completion_pos_bytes; 333 uint64_t peer_bytes_queued; 334 uint64_t bytes_transferred; 335 uint32_t target_queue_depth; 336 uint32_t cur_active_io; 337 uint8_t *extra_buf; 338 uint32_t extra_buf_len; 339 struct camdd_dev *peer_dev; 340 pthread_mutex_t mutex; 341 pthread_cond_t cond; 342 int kq; 343 344 int (*run)(struct camdd_dev *dev); 345 int (*fetch)(struct camdd_dev *dev); 346 347 /* 348 * Buffers that are available for I/O. Uses links. 349 */ 350 STAILQ_HEAD(,camdd_buf) free_queue; 351 352 /* 353 * Free indirect buffers. These are used for breaking a large 354 * buffer into multiple pieces. 355 */ 356 STAILQ_HEAD(,camdd_buf) free_indirect_queue; 357 358 /* 359 * Buffers that have been queued to the kernel. Uses links. 360 */ 361 STAILQ_HEAD(,camdd_buf) active_queue; 362 363 /* 364 * Will generally contain one of our buffers that is waiting for enough 365 * I/O from our partner thread to be able to execute. This will 366 * generally happen when our per-I/O-size is larger than the 367 * partner thread's per-I/O-size. Uses links. 368 */ 369 STAILQ_HEAD(,camdd_buf) pending_queue; 370 371 /* 372 * Number of buffers on the pending queue 373 */ 374 int num_pending_queue; 375 376 /* 377 * Buffers that are filled and ready to execute. This is used when 378 * our partner (reader) thread sends us blocks that are larger than 379 * our blocksize, and so we have to split them into multiple pieces. 380 */ 381 STAILQ_HEAD(,camdd_buf) run_queue; 382 383 /* 384 * Number of buffers on the run queue. 385 */ 386 int num_run_queue; 387 388 STAILQ_HEAD(,camdd_buf) reorder_queue; 389 390 int num_reorder_queue; 391 392 /* 393 * Buffers that have been queued to us by our partner thread 394 * (generally the reader thread) to be written out. Uses 395 * work_links. 396 */ 397 STAILQ_HEAD(,camdd_buf) work_queue; 398 399 /* 400 * Buffers that have been completed by our partner thread. Uses 401 * work_links. 402 */ 403 STAILQ_HEAD(,camdd_buf) peer_done_queue; 404 405 /* 406 * Number of buffers on the peer done queue. 407 */ 408 uint32_t num_peer_done_queue; 409 410 /* 411 * A list of buffers that we have queued to our peer thread. Uses 412 * links. 413 */ 414 STAILQ_HEAD(,camdd_buf) peer_work_queue; 415 416 /* 417 * Number of buffers on the peer work queue. 418 */ 419 uint32_t num_peer_work_queue; 420 }; 421 422 static sem_t camdd_sem; 423 static sig_atomic_t need_exit = 0; 424 static sig_atomic_t error_exit = 0; 425 static sig_atomic_t need_status = 0; 426 427 #ifndef min 428 #define min(a, b) (a < b) ? a : b 429 #endif 430 431 /* 432 * XXX KDM private copy of timespecsub(). This is normally defined in 433 * sys/time.h, but is only enabled in the kernel. If that definition is 434 * enabled in userland, it breaks the build of libnetbsd. 435 */ 436 #ifndef timespecsub 437 #define timespecsub(vvp, uvp) \ 438 do { \ 439 (vvp)->tv_sec -= (uvp)->tv_sec; \ 440 (vvp)->tv_nsec -= (uvp)->tv_nsec; \ 441 if ((vvp)->tv_nsec < 0) { \ 442 (vvp)->tv_sec--; \ 443 (vvp)->tv_nsec += 1000000000; \ 444 } \ 445 } while (0) 446 #endif 447 448 449 /* Generically useful offsets into the peripheral private area */ 450 #define ppriv_ptr0 periph_priv.entries[0].ptr 451 #define ppriv_ptr1 periph_priv.entries[1].ptr 452 #define ppriv_field0 periph_priv.entries[0].field 453 #define ppriv_field1 periph_priv.entries[1].field 454 455 #define ccb_buf ppriv_ptr0 456 457 #define CAMDD_FILE_DEFAULT_BLOCK 524288 458 #define CAMDD_FILE_DEFAULT_DEPTH 1 459 #define CAMDD_PASS_MAX_BLOCK 1048576 460 #define CAMDD_PASS_DEFAULT_DEPTH 6 461 #define CAMDD_PASS_RW_TIMEOUT 60 * 1000 462 463 static int parse_btl(char *tstr, int *bus, int *target, int *lun, 464 camdd_argmask *arglst); 465 void camdd_free_dev(struct camdd_dev *dev); 466 struct camdd_dev *camdd_alloc_dev(camdd_dev_type dev_type, 467 struct kevent *new_ke, int num_ke, 468 int retry_count, int timeout); 469 static struct camdd_buf *camdd_alloc_buf(struct camdd_dev *dev, 470 camdd_buf_type buf_type); 471 void camdd_release_buf(struct camdd_buf *buf); 472 struct camdd_buf *camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type); 473 int camdd_buf_sg_create(struct camdd_buf *buf, int iovec, 474 uint32_t sector_size, uint32_t *num_sectors_used, 475 int *double_buf_needed); 476 uint32_t camdd_buf_get_len(struct camdd_buf *buf); 477 void camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf); 478 int camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize, 479 uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran); 480 struct camdd_dev *camdd_probe_file(int fd, struct camdd_io_opts *io_opts, 481 int retry_count, int timeout); 482 struct camdd_dev *camdd_probe_pass(struct cam_device *cam_dev, 483 struct camdd_io_opts *io_opts, 484 camdd_argmask arglist, int probe_retry_count, 485 int probe_timeout, int io_retry_count, 486 int io_timeout); 487 void *camdd_file_worker(void *arg); 488 camdd_buf_status camdd_ccb_status(union ccb *ccb); 489 int camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf); 490 int camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf); 491 void camdd_peer_done(struct camdd_buf *buf); 492 void camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf, 493 int *error_count); 494 int camdd_pass_fetch(struct camdd_dev *dev); 495 int camdd_file_run(struct camdd_dev *dev); 496 int camdd_pass_run(struct camdd_dev *dev); 497 int camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len); 498 int camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf); 499 void camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth, 500 uint32_t *peer_depth, uint32_t *our_bytes, 501 uint32_t *peer_bytes); 502 void *camdd_worker(void *arg); 503 void camdd_sig_handler(int sig); 504 void camdd_print_status(struct camdd_dev *camdd_dev, 505 struct camdd_dev *other_dev, 506 struct timespec *start_time); 507 int camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts, 508 uint64_t max_io, int retry_count, int timeout); 509 int camdd_parse_io_opts(char *args, int is_write, 510 struct camdd_io_opts *io_opts); 511 void usage(void); 512 513 /* 514 * Parse out a bus, or a bus, target and lun in the following 515 * format: 516 * bus 517 * bus:target 518 * bus:target:lun 519 * 520 * Returns the number of parsed components, or 0. 521 */ 522 static int 523 parse_btl(char *tstr, int *bus, int *target, int *lun, camdd_argmask *arglst) 524 { 525 char *tmpstr; 526 int convs = 0; 527 528 while (isspace(*tstr) && (*tstr != '\0')) 529 tstr++; 530 531 tmpstr = (char *)strtok(tstr, ":"); 532 if ((tmpstr != NULL) && (*tmpstr != '\0')) { 533 *bus = strtol(tmpstr, NULL, 0); 534 *arglst |= CAMDD_ARG_BUS; 535 convs++; 536 tmpstr = (char *)strtok(NULL, ":"); 537 if ((tmpstr != NULL) && (*tmpstr != '\0')) { 538 *target = strtol(tmpstr, NULL, 0); 539 *arglst |= CAMDD_ARG_TARGET; 540 convs++; 541 tmpstr = (char *)strtok(NULL, ":"); 542 if ((tmpstr != NULL) && (*tmpstr != '\0')) { 543 *lun = strtol(tmpstr, NULL, 0); 544 *arglst |= CAMDD_ARG_LUN; 545 convs++; 546 } 547 } 548 } 549 550 return convs; 551 } 552 553 /* 554 * XXX KDM clean up and free all of the buffers on the queue! 555 */ 556 void 557 camdd_free_dev(struct camdd_dev *dev) 558 { 559 if (dev == NULL) 560 return; 561 562 switch (dev->dev_type) { 563 case CAMDD_DEV_FILE: { 564 struct camdd_dev_file *file_dev = &dev->dev_spec.file; 565 566 if (file_dev->fd != -1) 567 close(file_dev->fd); 568 free(file_dev->tmp_buf); 569 break; 570 } 571 case CAMDD_DEV_PASS: { 572 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass; 573 574 if (pass_dev->dev != NULL) 575 cam_close_device(pass_dev->dev); 576 break; 577 } 578 default: 579 break; 580 } 581 582 free(dev); 583 } 584 585 struct camdd_dev * 586 camdd_alloc_dev(camdd_dev_type dev_type, struct kevent *new_ke, int num_ke, 587 int retry_count, int timeout) 588 { 589 struct camdd_dev *dev = NULL; 590 struct kevent *ke; 591 size_t ke_size; 592 int retval = 0; 593 594 dev = malloc(sizeof(*dev)); 595 if (dev == NULL) { 596 warn("%s: unable to malloc %zu bytes", __func__, sizeof(*dev)); 597 goto bailout; 598 } 599 600 bzero(dev, sizeof(*dev)); 601 602 dev->dev_type = dev_type; 603 dev->io_timeout = timeout; 604 dev->retry_count = retry_count; 605 STAILQ_INIT(&dev->free_queue); 606 STAILQ_INIT(&dev->free_indirect_queue); 607 STAILQ_INIT(&dev->active_queue); 608 STAILQ_INIT(&dev->pending_queue); 609 STAILQ_INIT(&dev->run_queue); 610 STAILQ_INIT(&dev->reorder_queue); 611 STAILQ_INIT(&dev->work_queue); 612 STAILQ_INIT(&dev->peer_done_queue); 613 STAILQ_INIT(&dev->peer_work_queue); 614 retval = pthread_mutex_init(&dev->mutex, NULL); 615 if (retval != 0) { 616 warnc(retval, "%s: failed to initialize mutex", __func__); 617 goto bailout; 618 } 619 620 retval = pthread_cond_init(&dev->cond, NULL); 621 if (retval != 0) { 622 warnc(retval, "%s: failed to initialize condition variable", 623 __func__); 624 goto bailout; 625 } 626 627 dev->kq = kqueue(); 628 if (dev->kq == -1) { 629 warn("%s: Unable to create kqueue", __func__); 630 goto bailout; 631 } 632 633 ke_size = sizeof(struct kevent) * (num_ke + 4); 634 ke = malloc(ke_size); 635 if (ke == NULL) { 636 warn("%s: unable to malloc %zu bytes", __func__, ke_size); 637 goto bailout; 638 } 639 bzero(ke, ke_size); 640 if (num_ke > 0) 641 bcopy(new_ke, ke, num_ke * sizeof(struct kevent)); 642 643 EV_SET(&ke[num_ke++], (uintptr_t)&dev->work_queue, EVFILT_USER, 644 EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0); 645 EV_SET(&ke[num_ke++], (uintptr_t)&dev->peer_done_queue, EVFILT_USER, 646 EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0); 647 EV_SET(&ke[num_ke++], SIGINFO, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0); 648 EV_SET(&ke[num_ke++], SIGINT, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0); 649 650 retval = kevent(dev->kq, ke, num_ke, NULL, 0, NULL); 651 if (retval == -1) { 652 warn("%s: Unable to register kevents", __func__); 653 goto bailout; 654 } 655 656 657 return (dev); 658 659 bailout: 660 free(dev); 661 662 return (NULL); 663 } 664 665 static struct camdd_buf * 666 camdd_alloc_buf(struct camdd_dev *dev, camdd_buf_type buf_type) 667 { 668 struct camdd_buf *buf = NULL; 669 uint8_t *data_ptr = NULL; 670 671 /* 672 * We only need to allocate data space for data buffers. 673 */ 674 switch (buf_type) { 675 case CAMDD_BUF_DATA: 676 data_ptr = malloc(dev->blocksize); 677 if (data_ptr == NULL) { 678 warn("unable to allocate %u bytes", dev->blocksize); 679 goto bailout_error; 680 } 681 break; 682 default: 683 break; 684 } 685 686 buf = malloc(sizeof(*buf)); 687 if (buf == NULL) { 688 warn("unable to allocate %zu bytes", sizeof(*buf)); 689 goto bailout_error; 690 } 691 692 bzero(buf, sizeof(*buf)); 693 buf->buf_type = buf_type; 694 buf->dev = dev; 695 switch (buf_type) { 696 case CAMDD_BUF_DATA: { 697 struct camdd_buf_data *data; 698 699 data = &buf->buf_type_spec.data; 700 701 data->alloc_len = dev->blocksize; 702 data->buf = data_ptr; 703 break; 704 } 705 case CAMDD_BUF_INDIRECT: 706 break; 707 default: 708 break; 709 } 710 STAILQ_INIT(&buf->src_list); 711 712 return (buf); 713 714 bailout_error: 715 free(data_ptr); 716 717 return (NULL); 718 } 719 720 void 721 camdd_release_buf(struct camdd_buf *buf) 722 { 723 struct camdd_dev *dev; 724 725 dev = buf->dev; 726 727 switch (buf->buf_type) { 728 case CAMDD_BUF_DATA: { 729 struct camdd_buf_data *data; 730 731 data = &buf->buf_type_spec.data; 732 733 if (data->segs != NULL) { 734 if (data->extra_buf != 0) { 735 void *extra_buf; 736 737 extra_buf = (void *) 738 data->segs[data->sg_count - 1].ds_addr; 739 free(extra_buf); 740 data->extra_buf = 0; 741 } 742 free(data->segs); 743 data->segs = NULL; 744 data->sg_count = 0; 745 } else if (data->iovec != NULL) { 746 if (data->extra_buf != 0) { 747 free(data->iovec[data->sg_count - 1].iov_base); 748 data->extra_buf = 0; 749 } 750 free(data->iovec); 751 data->iovec = NULL; 752 data->sg_count = 0; 753 } 754 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 755 break; 756 } 757 case CAMDD_BUF_INDIRECT: 758 STAILQ_INSERT_TAIL(&dev->free_indirect_queue, buf, links); 759 break; 760 default: 761 err(1, "%s: Invalid buffer type %d for released buffer", 762 __func__, buf->buf_type); 763 break; 764 } 765 } 766 767 struct camdd_buf * 768 camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type) 769 { 770 struct camdd_buf *buf = NULL; 771 772 switch (buf_type) { 773 case CAMDD_BUF_DATA: 774 buf = STAILQ_FIRST(&dev->free_queue); 775 if (buf != NULL) { 776 struct camdd_buf_data *data; 777 uint8_t *data_ptr; 778 uint32_t alloc_len; 779 780 STAILQ_REMOVE_HEAD(&dev->free_queue, links); 781 data = &buf->buf_type_spec.data; 782 data_ptr = data->buf; 783 alloc_len = data->alloc_len; 784 bzero(buf, sizeof(*buf)); 785 data->buf = data_ptr; 786 data->alloc_len = alloc_len; 787 } 788 break; 789 case CAMDD_BUF_INDIRECT: 790 buf = STAILQ_FIRST(&dev->free_indirect_queue); 791 if (buf != NULL) { 792 STAILQ_REMOVE_HEAD(&dev->free_indirect_queue, links); 793 794 bzero(buf, sizeof(*buf)); 795 } 796 break; 797 default: 798 warnx("Unknown buffer type %d requested", buf_type); 799 break; 800 } 801 802 803 if (buf == NULL) 804 return (camdd_alloc_buf(dev, buf_type)); 805 else { 806 STAILQ_INIT(&buf->src_list); 807 buf->dev = dev; 808 buf->buf_type = buf_type; 809 810 return (buf); 811 } 812 } 813 814 int 815 camdd_buf_sg_create(struct camdd_buf *buf, int iovec, uint32_t sector_size, 816 uint32_t *num_sectors_used, int *double_buf_needed) 817 { 818 struct camdd_buf *tmp_buf; 819 struct camdd_buf_data *data; 820 uint8_t *extra_buf = NULL; 821 size_t extra_buf_len = 0; 822 int i, retval = 0; 823 824 data = &buf->buf_type_spec.data; 825 826 data->sg_count = buf->src_count; 827 /* 828 * Compose a scatter/gather list from all of the buffers in the list. 829 * If the length of the buffer isn't a multiple of the sector size, 830 * we'll have to add an extra buffer. This should only happen 831 * at the end of a transfer. 832 */ 833 if ((data->fill_len % sector_size) != 0) { 834 extra_buf_len = sector_size - (data->fill_len % sector_size); 835 extra_buf = calloc(extra_buf_len, 1); 836 if (extra_buf == NULL) { 837 warn("%s: unable to allocate %zu bytes for extra " 838 "buffer space", __func__, extra_buf_len); 839 retval = 1; 840 goto bailout; 841 } 842 data->extra_buf = 1; 843 data->sg_count++; 844 } 845 if (iovec == 0) { 846 data->segs = calloc(data->sg_count, sizeof(bus_dma_segment_t)); 847 if (data->segs == NULL) { 848 warn("%s: unable to allocate %zu bytes for S/G list", 849 __func__, sizeof(bus_dma_segment_t) * 850 data->sg_count); 851 retval = 1; 852 goto bailout; 853 } 854 855 } else { 856 data->iovec = calloc(data->sg_count, sizeof(struct iovec)); 857 if (data->iovec == NULL) { 858 warn("%s: unable to allocate %zu bytes for S/G list", 859 __func__, sizeof(struct iovec) * data->sg_count); 860 retval = 1; 861 goto bailout; 862 } 863 } 864 865 for (i = 0, tmp_buf = STAILQ_FIRST(&buf->src_list); 866 i < buf->src_count && tmp_buf != NULL; i++, 867 tmp_buf = STAILQ_NEXT(tmp_buf, src_links)) { 868 869 if (tmp_buf->buf_type == CAMDD_BUF_DATA) { 870 struct camdd_buf_data *tmp_data; 871 872 tmp_data = &tmp_buf->buf_type_spec.data; 873 if (iovec == 0) { 874 data->segs[i].ds_addr = 875 (bus_addr_t) tmp_data->buf; 876 data->segs[i].ds_len = tmp_data->fill_len - 877 tmp_data->resid; 878 } else { 879 data->iovec[i].iov_base = tmp_data->buf; 880 data->iovec[i].iov_len = tmp_data->fill_len - 881 tmp_data->resid; 882 } 883 if (((tmp_data->fill_len - tmp_data->resid) % 884 sector_size) != 0) 885 *double_buf_needed = 1; 886 } else { 887 struct camdd_buf_indirect *tmp_ind; 888 889 tmp_ind = &tmp_buf->buf_type_spec.indirect; 890 if (iovec == 0) { 891 data->segs[i].ds_addr = 892 (bus_addr_t)tmp_ind->start_ptr; 893 data->segs[i].ds_len = tmp_ind->len; 894 } else { 895 data->iovec[i].iov_base = tmp_ind->start_ptr; 896 data->iovec[i].iov_len = tmp_ind->len; 897 } 898 if ((tmp_ind->len % sector_size) != 0) 899 *double_buf_needed = 1; 900 } 901 } 902 903 if (extra_buf != NULL) { 904 if (iovec == 0) { 905 data->segs[i].ds_addr = (bus_addr_t)extra_buf; 906 data->segs[i].ds_len = extra_buf_len; 907 } else { 908 data->iovec[i].iov_base = extra_buf; 909 data->iovec[i].iov_len = extra_buf_len; 910 } 911 i++; 912 } 913 if ((tmp_buf != NULL) || (i != data->sg_count)) { 914 warnx("buffer source count does not match " 915 "number of buffers in list!"); 916 retval = 1; 917 goto bailout; 918 } 919 920 bailout: 921 if (retval == 0) { 922 *num_sectors_used = (data->fill_len + extra_buf_len) / 923 sector_size; 924 } 925 return (retval); 926 } 927 928 uint32_t 929 camdd_buf_get_len(struct camdd_buf *buf) 930 { 931 uint32_t len = 0; 932 933 if (buf->buf_type != CAMDD_BUF_DATA) { 934 struct camdd_buf_indirect *indirect; 935 936 indirect = &buf->buf_type_spec.indirect; 937 len = indirect->len; 938 } else { 939 struct camdd_buf_data *data; 940 941 data = &buf->buf_type_spec.data; 942 len = data->fill_len; 943 } 944 945 return (len); 946 } 947 948 void 949 camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf) 950 { 951 struct camdd_buf_data *data; 952 953 assert(buf->buf_type == CAMDD_BUF_DATA); 954 955 data = &buf->buf_type_spec.data; 956 957 STAILQ_INSERT_TAIL(&buf->src_list, child_buf, src_links); 958 buf->src_count++; 959 960 data->fill_len += camdd_buf_get_len(child_buf); 961 } 962 963 typedef enum { 964 CAMDD_TS_MAX_BLK, 965 CAMDD_TS_MIN_BLK, 966 CAMDD_TS_BLK_GRAN, 967 CAMDD_TS_EFF_IOSIZE 968 } camdd_status_item_index; 969 970 static struct camdd_status_items { 971 const char *name; 972 struct mt_status_entry *entry; 973 } req_status_items[] = { 974 { "max_blk", NULL }, 975 { "min_blk", NULL }, 976 { "blk_gran", NULL }, 977 { "max_effective_iosize", NULL } 978 }; 979 980 int 981 camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize, 982 uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran) 983 { 984 struct mt_status_data status_data; 985 char *xml_str = NULL; 986 unsigned int i; 987 int retval = 0; 988 989 retval = mt_get_xml_str(fd, MTIOCEXTGET, &xml_str); 990 if (retval != 0) 991 err(1, "Couldn't get XML string from %s", filename); 992 993 retval = mt_get_status(xml_str, &status_data); 994 if (retval != XML_STATUS_OK) { 995 warn("couldn't get status for %s", filename); 996 retval = 1; 997 goto bailout; 998 } else 999 retval = 0; 1000 1001 if (status_data.error != 0) { 1002 warnx("%s", status_data.error_str); 1003 retval = 1; 1004 goto bailout; 1005 } 1006 1007 for (i = 0; i < sizeof(req_status_items) / 1008 sizeof(req_status_items[0]); i++) { 1009 char *name; 1010 1011 name = __DECONST(char *, req_status_items[i].name); 1012 req_status_items[i].entry = mt_status_entry_find(&status_data, 1013 name); 1014 if (req_status_items[i].entry == NULL) { 1015 errx(1, "Cannot find status entry %s", 1016 req_status_items[i].name); 1017 } 1018 } 1019 1020 *max_iosize = req_status_items[CAMDD_TS_EFF_IOSIZE].entry->value_unsigned; 1021 *max_blk= req_status_items[CAMDD_TS_MAX_BLK].entry->value_unsigned; 1022 *min_blk= req_status_items[CAMDD_TS_MIN_BLK].entry->value_unsigned; 1023 *blk_gran = req_status_items[CAMDD_TS_BLK_GRAN].entry->value_unsigned; 1024 bailout: 1025 1026 free(xml_str); 1027 mt_status_free(&status_data); 1028 1029 return (retval); 1030 } 1031 1032 struct camdd_dev * 1033 camdd_probe_file(int fd, struct camdd_io_opts *io_opts, int retry_count, 1034 int timeout) 1035 { 1036 struct camdd_dev *dev = NULL; 1037 struct camdd_dev_file *file_dev; 1038 uint64_t blocksize = io_opts->blocksize; 1039 1040 dev = camdd_alloc_dev(CAMDD_DEV_FILE, NULL, 0, retry_count, timeout); 1041 if (dev == NULL) 1042 goto bailout; 1043 1044 file_dev = &dev->dev_spec.file; 1045 file_dev->fd = fd; 1046 strlcpy(file_dev->filename, io_opts->dev_name, 1047 sizeof(file_dev->filename)); 1048 strlcpy(dev->device_name, io_opts->dev_name, sizeof(dev->device_name)); 1049 if (blocksize == 0) 1050 dev->blocksize = CAMDD_FILE_DEFAULT_BLOCK; 1051 else 1052 dev->blocksize = blocksize; 1053 1054 if ((io_opts->queue_depth != 0) 1055 && (io_opts->queue_depth != 1)) { 1056 warnx("Queue depth %ju for %s ignored, only 1 outstanding " 1057 "command supported", (uintmax_t)io_opts->queue_depth, 1058 io_opts->dev_name); 1059 } 1060 dev->target_queue_depth = CAMDD_FILE_DEFAULT_DEPTH; 1061 dev->run = camdd_file_run; 1062 dev->fetch = NULL; 1063 1064 /* 1065 * We can effectively access files on byte boundaries. We'll reset 1066 * this for devices like disks that can be accessed on sector 1067 * boundaries. 1068 */ 1069 dev->sector_size = 1; 1070 1071 if ((fd != STDIN_FILENO) 1072 && (fd != STDOUT_FILENO)) { 1073 int retval; 1074 1075 retval = fstat(fd, &file_dev->sb); 1076 if (retval != 0) { 1077 warn("Cannot stat %s", dev->device_name); 1078 goto bailout_error; 1079 } 1080 if (S_ISREG(file_dev->sb.st_mode)) { 1081 file_dev->file_type = CAMDD_FILE_REG; 1082 } else if (S_ISCHR(file_dev->sb.st_mode)) { 1083 int type; 1084 1085 if (ioctl(fd, FIODTYPE, &type) == -1) 1086 err(1, "FIODTYPE ioctl failed on %s", 1087 dev->device_name); 1088 else { 1089 if (type & D_TAPE) 1090 file_dev->file_type = CAMDD_FILE_TAPE; 1091 else if (type & D_DISK) 1092 file_dev->file_type = CAMDD_FILE_DISK; 1093 else if (type & D_MEM) 1094 file_dev->file_type = CAMDD_FILE_MEM; 1095 else if (type & D_TTY) 1096 file_dev->file_type = CAMDD_FILE_TTY; 1097 } 1098 } else if (S_ISDIR(file_dev->sb.st_mode)) { 1099 errx(1, "cannot operate on directory %s", 1100 dev->device_name); 1101 } else if (S_ISFIFO(file_dev->sb.st_mode)) { 1102 file_dev->file_type = CAMDD_FILE_PIPE; 1103 } else 1104 errx(1, "Cannot determine file type for %s", 1105 dev->device_name); 1106 1107 switch (file_dev->file_type) { 1108 case CAMDD_FILE_REG: 1109 if (file_dev->sb.st_size != 0) 1110 dev->max_sector = file_dev->sb.st_size - 1; 1111 else 1112 dev->max_sector = 0; 1113 file_dev->file_flags |= CAMDD_FF_CAN_SEEK; 1114 break; 1115 case CAMDD_FILE_TAPE: { 1116 uint64_t max_iosize, max_blk, min_blk, blk_gran; 1117 /* 1118 * Check block limits and maximum effective iosize. 1119 * Make sure the blocksize is within the block 1120 * limits (and a multiple of the minimum blocksize) 1121 * and that the blocksize is <= maximum effective 1122 * iosize. 1123 */ 1124 retval = camdd_probe_tape(fd, dev->device_name, 1125 &max_iosize, &max_blk, &min_blk, &blk_gran); 1126 if (retval != 0) 1127 errx(1, "Unable to probe tape %s", 1128 dev->device_name); 1129 1130 /* 1131 * The blocksize needs to be <= the maximum 1132 * effective I/O size of the tape device. Note 1133 * that this also takes into account the maximum 1134 * blocksize reported by READ BLOCK LIMITS. 1135 */ 1136 if (dev->blocksize > max_iosize) { 1137 warnx("Blocksize %u too big for %s, limiting " 1138 "to %ju", dev->blocksize, dev->device_name, 1139 max_iosize); 1140 dev->blocksize = max_iosize; 1141 } 1142 1143 /* 1144 * The blocksize needs to be at least min_blk; 1145 */ 1146 if (dev->blocksize < min_blk) { 1147 warnx("Blocksize %u too small for %s, " 1148 "increasing to %ju", dev->blocksize, 1149 dev->device_name, min_blk); 1150 dev->blocksize = min_blk; 1151 } 1152 1153 /* 1154 * And the blocksize needs to be a multiple of 1155 * the block granularity. 1156 */ 1157 if ((blk_gran != 0) 1158 && (dev->blocksize % (1 << blk_gran))) { 1159 warnx("Blocksize %u for %s not a multiple of " 1160 "%d, adjusting to %d", dev->blocksize, 1161 dev->device_name, (1 << blk_gran), 1162 dev->blocksize & ~((1 << blk_gran) - 1)); 1163 dev->blocksize &= ~((1 << blk_gran) - 1); 1164 } 1165 1166 if (dev->blocksize == 0) { 1167 errx(1, "Unable to derive valid blocksize for " 1168 "%s", dev->device_name); 1169 } 1170 1171 /* 1172 * For tape drives, set the sector size to the 1173 * blocksize so that we make sure not to write 1174 * less than the blocksize out to the drive. 1175 */ 1176 dev->sector_size = dev->blocksize; 1177 break; 1178 } 1179 case CAMDD_FILE_DISK: { 1180 off_t media_size; 1181 unsigned int sector_size; 1182 1183 file_dev->file_flags |= CAMDD_FF_CAN_SEEK; 1184 1185 if (ioctl(fd, DIOCGSECTORSIZE, §or_size) == -1) { 1186 err(1, "DIOCGSECTORSIZE ioctl failed on %s", 1187 dev->device_name); 1188 } 1189 1190 if (sector_size == 0) { 1191 errx(1, "DIOCGSECTORSIZE ioctl returned " 1192 "invalid sector size %u for %s", 1193 sector_size, dev->device_name); 1194 } 1195 1196 if (ioctl(fd, DIOCGMEDIASIZE, &media_size) == -1) { 1197 err(1, "DIOCGMEDIASIZE ioctl failed on %s", 1198 dev->device_name); 1199 } 1200 1201 if (media_size == 0) { 1202 errx(1, "DIOCGMEDIASIZE ioctl returned " 1203 "invalid media size %ju for %s", 1204 (uintmax_t)media_size, dev->device_name); 1205 } 1206 1207 if (dev->blocksize % sector_size) { 1208 errx(1, "%s blocksize %u not a multiple of " 1209 "sector size %u", dev->device_name, 1210 dev->blocksize, sector_size); 1211 } 1212 1213 dev->sector_size = sector_size; 1214 dev->max_sector = (media_size / sector_size) - 1; 1215 break; 1216 } 1217 case CAMDD_FILE_MEM: 1218 file_dev->file_flags |= CAMDD_FF_CAN_SEEK; 1219 break; 1220 default: 1221 break; 1222 } 1223 } 1224 1225 if ((io_opts->offset != 0) 1226 && ((file_dev->file_flags & CAMDD_FF_CAN_SEEK) == 0)) { 1227 warnx("Offset %ju specified for %s, but we cannot seek on %s", 1228 io_opts->offset, io_opts->dev_name, io_opts->dev_name); 1229 goto bailout_error; 1230 } 1231 #if 0 1232 else if ((io_opts->offset != 0) 1233 && ((io_opts->offset % dev->sector_size) != 0)) { 1234 warnx("Offset %ju for %s is not a multiple of the " 1235 "sector size %u", io_opts->offset, 1236 io_opts->dev_name, dev->sector_size); 1237 goto bailout_error; 1238 } else { 1239 dev->start_offset_bytes = io_opts->offset; 1240 } 1241 #endif 1242 1243 bailout: 1244 return (dev); 1245 1246 bailout_error: 1247 camdd_free_dev(dev); 1248 return (NULL); 1249 } 1250 1251 /* 1252 * Need to implement this. Do a basic probe: 1253 * - Check the inquiry data, make sure we're talking to a device that we 1254 * can reasonably expect to talk to -- direct, RBC, CD, WORM. 1255 * - Send a test unit ready, make sure the device is available. 1256 * - Get the capacity and block size. 1257 */ 1258 struct camdd_dev * 1259 camdd_probe_pass(struct cam_device *cam_dev, struct camdd_io_opts *io_opts, 1260 camdd_argmask arglist, int probe_retry_count, 1261 int probe_timeout, int io_retry_count, int io_timeout) 1262 { 1263 union ccb *ccb; 1264 uint64_t maxsector; 1265 uint32_t cpi_maxio, max_iosize, pass_numblocks; 1266 uint32_t block_len; 1267 struct scsi_read_capacity_data rcap; 1268 struct scsi_read_capacity_data_long rcaplong; 1269 struct camdd_dev *dev; 1270 struct camdd_dev_pass *pass_dev; 1271 struct kevent ke; 1272 int scsi_dev_type; 1273 1274 dev = NULL; 1275 1276 scsi_dev_type = SID_TYPE(&cam_dev->inq_data); 1277 maxsector = 0; 1278 block_len = 0; 1279 1280 /* 1281 * For devices that support READ CAPACITY, we'll attempt to get the 1282 * capacity. Otherwise, we really don't support tape or other 1283 * devices via SCSI passthrough, so just return an error in that case. 1284 */ 1285 switch (scsi_dev_type) { 1286 case T_DIRECT: 1287 case T_WORM: 1288 case T_CDROM: 1289 case T_OPTICAL: 1290 case T_RBC: 1291 case T_ZBC_HM: 1292 break; 1293 default: 1294 errx(1, "Unsupported SCSI device type %d", scsi_dev_type); 1295 break; /*NOTREACHED*/ 1296 } 1297 1298 ccb = cam_getccb(cam_dev); 1299 1300 if (ccb == NULL) { 1301 warnx("%s: error allocating ccb", __func__); 1302 goto bailout; 1303 } 1304 1305 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->csio); 1306 1307 scsi_read_capacity(&ccb->csio, 1308 /*retries*/ probe_retry_count, 1309 /*cbfcnp*/ NULL, 1310 /*tag_action*/ MSG_SIMPLE_Q_TAG, 1311 &rcap, 1312 SSD_FULL_SIZE, 1313 /*timeout*/ probe_timeout ? probe_timeout : 5000); 1314 1315 /* Disable freezing the device queue */ 1316 ccb->ccb_h.flags |= CAM_DEV_QFRZDIS; 1317 1318 if (arglist & CAMDD_ARG_ERR_RECOVER) 1319 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER; 1320 1321 if (cam_send_ccb(cam_dev, ccb) < 0) { 1322 warn("error sending READ CAPACITY command"); 1323 1324 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, 1325 CAM_EPF_ALL, stderr); 1326 1327 goto bailout; 1328 } 1329 1330 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1331 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr); 1332 goto bailout; 1333 } 1334 1335 maxsector = scsi_4btoul(rcap.addr); 1336 block_len = scsi_4btoul(rcap.length); 1337 1338 /* 1339 * A last block of 2^32-1 means that the true capacity is over 2TB, 1340 * and we need to issue the long READ CAPACITY to get the real 1341 * capacity. Otherwise, we're all set. 1342 */ 1343 if (maxsector != 0xffffffff) 1344 goto rcap_done; 1345 1346 scsi_read_capacity_16(&ccb->csio, 1347 /*retries*/ probe_retry_count, 1348 /*cbfcnp*/ NULL, 1349 /*tag_action*/ MSG_SIMPLE_Q_TAG, 1350 /*lba*/ 0, 1351 /*reladdr*/ 0, 1352 /*pmi*/ 0, 1353 (uint8_t *)&rcaplong, 1354 sizeof(rcaplong), 1355 /*sense_len*/ SSD_FULL_SIZE, 1356 /*timeout*/ probe_timeout ? probe_timeout : 5000); 1357 1358 /* Disable freezing the device queue */ 1359 ccb->ccb_h.flags |= CAM_DEV_QFRZDIS; 1360 1361 if (arglist & CAMDD_ARG_ERR_RECOVER) 1362 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER; 1363 1364 if (cam_send_ccb(cam_dev, ccb) < 0) { 1365 warn("error sending READ CAPACITY (16) command"); 1366 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, 1367 CAM_EPF_ALL, stderr); 1368 goto bailout; 1369 } 1370 1371 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1372 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr); 1373 goto bailout; 1374 } 1375 1376 maxsector = scsi_8btou64(rcaplong.addr); 1377 block_len = scsi_4btoul(rcaplong.length); 1378 1379 rcap_done: 1380 if (block_len == 0) { 1381 warnx("Sector size for %s%u is 0, cannot continue", 1382 cam_dev->device_name, cam_dev->dev_unit_num); 1383 goto bailout_error; 1384 } 1385 1386 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->cpi); 1387 1388 ccb->ccb_h.func_code = XPT_PATH_INQ; 1389 ccb->ccb_h.flags = CAM_DIR_NONE; 1390 ccb->ccb_h.retry_count = 1; 1391 1392 if (cam_send_ccb(cam_dev, ccb) < 0) { 1393 warn("error sending XPT_PATH_INQ CCB"); 1394 1395 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, 1396 CAM_EPF_ALL, stderr); 1397 goto bailout; 1398 } 1399 1400 EV_SET(&ke, cam_dev->fd, EVFILT_READ, EV_ADD|EV_ENABLE, 0, 0, 0); 1401 1402 dev = camdd_alloc_dev(CAMDD_DEV_PASS, &ke, 1, io_retry_count, 1403 io_timeout); 1404 if (dev == NULL) 1405 goto bailout; 1406 1407 pass_dev = &dev->dev_spec.pass; 1408 pass_dev->scsi_dev_type = scsi_dev_type; 1409 pass_dev->dev = cam_dev; 1410 pass_dev->max_sector = maxsector; 1411 pass_dev->block_len = block_len; 1412 pass_dev->cpi_maxio = ccb->cpi.maxio; 1413 snprintf(dev->device_name, sizeof(dev->device_name), "%s%u", 1414 pass_dev->dev->device_name, pass_dev->dev->dev_unit_num); 1415 dev->sector_size = block_len; 1416 dev->max_sector = maxsector; 1417 1418 1419 /* 1420 * Determine the optimal blocksize to use for this device. 1421 */ 1422 1423 /* 1424 * If the controller has not specified a maximum I/O size, 1425 * just go with 128K as a somewhat conservative value. 1426 */ 1427 if (pass_dev->cpi_maxio == 0) 1428 cpi_maxio = 131072; 1429 else 1430 cpi_maxio = pass_dev->cpi_maxio; 1431 1432 /* 1433 * If the controller has a large maximum I/O size, limit it 1434 * to something smaller so that the kernel doesn't have trouble 1435 * allocating buffers to copy data in and out for us. 1436 * XXX KDM this is until we have unmapped I/O support in the kernel. 1437 */ 1438 max_iosize = min(cpi_maxio, CAMDD_PASS_MAX_BLOCK); 1439 1440 /* 1441 * If we weren't able to get a block size for some reason, 1442 * default to 512 bytes. 1443 */ 1444 block_len = pass_dev->block_len; 1445 if (block_len == 0) 1446 block_len = 512; 1447 1448 /* 1449 * Figure out how many blocksize chunks will fit in the 1450 * maximum I/O size. 1451 */ 1452 pass_numblocks = max_iosize / block_len; 1453 1454 /* 1455 * And finally, multiple the number of blocks by the LBA 1456 * length to get our maximum block size; 1457 */ 1458 dev->blocksize = pass_numblocks * block_len; 1459 1460 if (io_opts->blocksize != 0) { 1461 if ((io_opts->blocksize % dev->sector_size) != 0) { 1462 warnx("Blocksize %ju for %s is not a multiple of " 1463 "sector size %u", (uintmax_t)io_opts->blocksize, 1464 dev->device_name, dev->sector_size); 1465 goto bailout_error; 1466 } 1467 dev->blocksize = io_opts->blocksize; 1468 } 1469 dev->target_queue_depth = CAMDD_PASS_DEFAULT_DEPTH; 1470 if (io_opts->queue_depth != 0) 1471 dev->target_queue_depth = io_opts->queue_depth; 1472 1473 if (io_opts->offset != 0) { 1474 if (io_opts->offset > (dev->max_sector * dev->sector_size)) { 1475 warnx("Offset %ju is past the end of device %s", 1476 io_opts->offset, dev->device_name); 1477 goto bailout_error; 1478 } 1479 #if 0 1480 else if ((io_opts->offset % dev->sector_size) != 0) { 1481 warnx("Offset %ju for %s is not a multiple of the " 1482 "sector size %u", io_opts->offset, 1483 dev->device_name, dev->sector_size); 1484 goto bailout_error; 1485 } 1486 dev->start_offset_bytes = io_opts->offset; 1487 #endif 1488 } 1489 1490 dev->min_cmd_size = io_opts->min_cmd_size; 1491 1492 dev->run = camdd_pass_run; 1493 dev->fetch = camdd_pass_fetch; 1494 1495 bailout: 1496 cam_freeccb(ccb); 1497 1498 return (dev); 1499 1500 bailout_error: 1501 cam_freeccb(ccb); 1502 1503 camdd_free_dev(dev); 1504 1505 return (NULL); 1506 } 1507 1508 void * 1509 camdd_worker(void *arg) 1510 { 1511 struct camdd_dev *dev = arg; 1512 struct camdd_buf *buf; 1513 struct timespec ts, *kq_ts; 1514 1515 ts.tv_sec = 0; 1516 ts.tv_nsec = 0; 1517 1518 pthread_mutex_lock(&dev->mutex); 1519 1520 dev->flags |= CAMDD_DEV_FLAG_ACTIVE; 1521 1522 for (;;) { 1523 struct kevent ke; 1524 int retval = 0; 1525 1526 /* 1527 * XXX KDM check the reorder queue depth? 1528 */ 1529 if (dev->write_dev == 0) { 1530 uint32_t our_depth, peer_depth, peer_bytes, our_bytes; 1531 uint32_t target_depth = dev->target_queue_depth; 1532 uint32_t peer_target_depth = 1533 dev->peer_dev->target_queue_depth; 1534 uint32_t peer_blocksize = dev->peer_dev->blocksize; 1535 1536 camdd_get_depth(dev, &our_depth, &peer_depth, 1537 &our_bytes, &peer_bytes); 1538 1539 #if 0 1540 while (((our_depth < target_depth) 1541 && (peer_depth < peer_target_depth)) 1542 || ((peer_bytes + our_bytes) < 1543 (peer_blocksize * 2))) { 1544 #endif 1545 while (((our_depth + peer_depth) < 1546 (target_depth + peer_target_depth)) 1547 || ((peer_bytes + our_bytes) < 1548 (peer_blocksize * 3))) { 1549 1550 retval = camdd_queue(dev, NULL); 1551 if (retval == 1) 1552 break; 1553 else if (retval != 0) { 1554 error_exit = 1; 1555 goto bailout; 1556 } 1557 1558 camdd_get_depth(dev, &our_depth, &peer_depth, 1559 &our_bytes, &peer_bytes); 1560 } 1561 } 1562 /* 1563 * See if we have any I/O that is ready to execute. 1564 */ 1565 buf = STAILQ_FIRST(&dev->run_queue); 1566 if (buf != NULL) { 1567 while (dev->target_queue_depth > dev->cur_active_io) { 1568 retval = dev->run(dev); 1569 if (retval == -1) { 1570 dev->flags |= CAMDD_DEV_FLAG_EOF; 1571 error_exit = 1; 1572 break; 1573 } else if (retval != 0) { 1574 break; 1575 } 1576 } 1577 } 1578 1579 /* 1580 * We've reached EOF, or our partner has reached EOF. 1581 */ 1582 if ((dev->flags & CAMDD_DEV_FLAG_EOF) 1583 || (dev->flags & CAMDD_DEV_FLAG_PEER_EOF)) { 1584 if (dev->write_dev != 0) { 1585 if ((STAILQ_EMPTY(&dev->work_queue)) 1586 && (dev->num_run_queue == 0) 1587 && (dev->cur_active_io == 0)) { 1588 goto bailout; 1589 } 1590 } else { 1591 /* 1592 * If we're the reader, and the writer 1593 * got EOF, he is already done. If we got 1594 * the EOF, then we need to wait until 1595 * everything is flushed out for the writer. 1596 */ 1597 if (dev->flags & CAMDD_DEV_FLAG_PEER_EOF) { 1598 goto bailout; 1599 } else if ((dev->num_peer_work_queue == 0) 1600 && (dev->num_peer_done_queue == 0) 1601 && (dev->cur_active_io == 0) 1602 && (dev->num_run_queue == 0)) { 1603 goto bailout; 1604 } 1605 } 1606 /* 1607 * XXX KDM need to do something about the pending 1608 * queue and cleanup resources. 1609 */ 1610 } 1611 1612 if ((dev->write_dev == 0) 1613 && (dev->cur_active_io == 0) 1614 && (dev->peer_bytes_queued < dev->peer_dev->blocksize)) 1615 kq_ts = &ts; 1616 else 1617 kq_ts = NULL; 1618 1619 /* 1620 * Run kevent to see if there are events to process. 1621 */ 1622 pthread_mutex_unlock(&dev->mutex); 1623 retval = kevent(dev->kq, NULL, 0, &ke, 1, kq_ts); 1624 pthread_mutex_lock(&dev->mutex); 1625 if (retval == -1) { 1626 warn("%s: error returned from kevent",__func__); 1627 goto bailout; 1628 } else if (retval != 0) { 1629 switch (ke.filter) { 1630 case EVFILT_READ: 1631 if (dev->fetch != NULL) { 1632 retval = dev->fetch(dev); 1633 if (retval == -1) { 1634 error_exit = 1; 1635 goto bailout; 1636 } 1637 } 1638 break; 1639 case EVFILT_SIGNAL: 1640 /* 1641 * We register for this so we don't get 1642 * an error as a result of a SIGINFO or a 1643 * SIGINT. It will actually get handled 1644 * by the signal handler. If we get a 1645 * SIGINT, bail out without printing an 1646 * error message. Any other signals 1647 * will result in the error message above. 1648 */ 1649 if (ke.ident == SIGINT) 1650 goto bailout; 1651 break; 1652 case EVFILT_USER: 1653 retval = 0; 1654 /* 1655 * Check to see if the other thread has 1656 * queued any I/O for us to do. (In this 1657 * case we're the writer.) 1658 */ 1659 for (buf = STAILQ_FIRST(&dev->work_queue); 1660 buf != NULL; 1661 buf = STAILQ_FIRST(&dev->work_queue)) { 1662 STAILQ_REMOVE_HEAD(&dev->work_queue, 1663 work_links); 1664 retval = camdd_queue(dev, buf); 1665 /* 1666 * We keep going unless we get an 1667 * actual error. If we get EOF, we 1668 * still want to remove the buffers 1669 * from the queue and send the back 1670 * to the reader thread. 1671 */ 1672 if (retval == -1) { 1673 error_exit = 1; 1674 goto bailout; 1675 } else 1676 retval = 0; 1677 } 1678 1679 /* 1680 * Next check to see if the other thread has 1681 * queued any completed buffers back to us. 1682 * (In this case we're the reader.) 1683 */ 1684 for (buf = STAILQ_FIRST(&dev->peer_done_queue); 1685 buf != NULL; 1686 buf = STAILQ_FIRST(&dev->peer_done_queue)){ 1687 STAILQ_REMOVE_HEAD( 1688 &dev->peer_done_queue, work_links); 1689 dev->num_peer_done_queue--; 1690 camdd_peer_done(buf); 1691 } 1692 break; 1693 default: 1694 warnx("%s: unknown kevent filter %d", 1695 __func__, ke.filter); 1696 break; 1697 } 1698 } 1699 } 1700 1701 bailout: 1702 1703 dev->flags &= ~CAMDD_DEV_FLAG_ACTIVE; 1704 1705 /* XXX KDM cleanup resources here? */ 1706 1707 pthread_mutex_unlock(&dev->mutex); 1708 1709 need_exit = 1; 1710 sem_post(&camdd_sem); 1711 1712 return (NULL); 1713 } 1714 1715 /* 1716 * Simplistic translation of CCB status to our local status. 1717 */ 1718 camdd_buf_status 1719 camdd_ccb_status(union ccb *ccb) 1720 { 1721 camdd_buf_status status = CAMDD_STATUS_NONE; 1722 cam_status ccb_status; 1723 1724 ccb_status = ccb->ccb_h.status & CAM_STATUS_MASK; 1725 1726 switch (ccb_status) { 1727 case CAM_REQ_CMP: { 1728 if (ccb->csio.resid == 0) { 1729 status = CAMDD_STATUS_OK; 1730 } else if (ccb->csio.dxfer_len > ccb->csio.resid) { 1731 status = CAMDD_STATUS_SHORT_IO; 1732 } else { 1733 status = CAMDD_STATUS_EOF; 1734 } 1735 break; 1736 } 1737 case CAM_SCSI_STATUS_ERROR: { 1738 switch (ccb->csio.scsi_status) { 1739 case SCSI_STATUS_OK: 1740 case SCSI_STATUS_COND_MET: 1741 case SCSI_STATUS_INTERMED: 1742 case SCSI_STATUS_INTERMED_COND_MET: 1743 status = CAMDD_STATUS_OK; 1744 break; 1745 case SCSI_STATUS_CMD_TERMINATED: 1746 case SCSI_STATUS_CHECK_COND: 1747 case SCSI_STATUS_QUEUE_FULL: 1748 case SCSI_STATUS_BUSY: 1749 case SCSI_STATUS_RESERV_CONFLICT: 1750 default: 1751 status = CAMDD_STATUS_ERROR; 1752 break; 1753 } 1754 break; 1755 } 1756 default: 1757 status = CAMDD_STATUS_ERROR; 1758 break; 1759 } 1760 1761 return (status); 1762 } 1763 1764 /* 1765 * Queue a buffer to our peer's work thread for writing. 1766 * 1767 * Returns 0 for success, -1 for failure, 1 if the other thread exited. 1768 */ 1769 int 1770 camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf) 1771 { 1772 struct kevent ke; 1773 STAILQ_HEAD(, camdd_buf) local_queue; 1774 struct camdd_buf *buf1, *buf2; 1775 struct camdd_buf_data *data = NULL; 1776 uint64_t peer_bytes_queued = 0; 1777 int active = 1; 1778 int retval = 0; 1779 1780 STAILQ_INIT(&local_queue); 1781 1782 /* 1783 * Since we're the reader, we need to queue our I/O to the writer 1784 * in sequential order in order to make sure it gets written out 1785 * in sequential order. 1786 * 1787 * Check the next expected I/O starting offset. If this doesn't 1788 * match, put it on the reorder queue. 1789 */ 1790 if ((buf->lba * dev->sector_size) != dev->next_completion_pos_bytes) { 1791 1792 /* 1793 * If there is nothing on the queue, there is no sorting 1794 * needed. 1795 */ 1796 if (STAILQ_EMPTY(&dev->reorder_queue)) { 1797 STAILQ_INSERT_TAIL(&dev->reorder_queue, buf, links); 1798 dev->num_reorder_queue++; 1799 goto bailout; 1800 } 1801 1802 /* 1803 * Sort in ascending order by starting LBA. There should 1804 * be no identical LBAs. 1805 */ 1806 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL; 1807 buf1 = buf2) { 1808 buf2 = STAILQ_NEXT(buf1, links); 1809 if (buf->lba < buf1->lba) { 1810 /* 1811 * If we're less than the first one, then 1812 * we insert at the head of the list 1813 * because this has to be the first element 1814 * on the list. 1815 */ 1816 STAILQ_INSERT_HEAD(&dev->reorder_queue, 1817 buf, links); 1818 dev->num_reorder_queue++; 1819 break; 1820 } else if (buf->lba > buf1->lba) { 1821 if (buf2 == NULL) { 1822 STAILQ_INSERT_TAIL(&dev->reorder_queue, 1823 buf, links); 1824 dev->num_reorder_queue++; 1825 break; 1826 } else if (buf->lba < buf2->lba) { 1827 STAILQ_INSERT_AFTER(&dev->reorder_queue, 1828 buf1, buf, links); 1829 dev->num_reorder_queue++; 1830 break; 1831 } 1832 } else { 1833 errx(1, "Found buffers with duplicate LBA %ju!", 1834 buf->lba); 1835 } 1836 } 1837 goto bailout; 1838 } else { 1839 1840 /* 1841 * We're the next expected I/O completion, so put ourselves 1842 * on the local queue to be sent to the writer. We use 1843 * work_links here so that we can queue this to the 1844 * peer_work_queue before taking the buffer off of the 1845 * local_queue. 1846 */ 1847 dev->next_completion_pos_bytes += buf->len; 1848 STAILQ_INSERT_TAIL(&local_queue, buf, work_links); 1849 1850 /* 1851 * Go through the reorder queue looking for more sequential 1852 * I/O and add it to the local queue. 1853 */ 1854 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL; 1855 buf1 = STAILQ_FIRST(&dev->reorder_queue)) { 1856 /* 1857 * As soon as we see an I/O that is out of sequence, 1858 * we're done. 1859 */ 1860 if ((buf1->lba * dev->sector_size) != 1861 dev->next_completion_pos_bytes) 1862 break; 1863 1864 STAILQ_REMOVE_HEAD(&dev->reorder_queue, links); 1865 dev->num_reorder_queue--; 1866 STAILQ_INSERT_TAIL(&local_queue, buf1, work_links); 1867 dev->next_completion_pos_bytes += buf1->len; 1868 } 1869 } 1870 1871 /* 1872 * Setup the event to let the other thread know that it has work 1873 * pending. 1874 */ 1875 EV_SET(&ke, (uintptr_t)&dev->peer_dev->work_queue, EVFILT_USER, 0, 1876 NOTE_TRIGGER, 0, NULL); 1877 1878 /* 1879 * Put this on our shadow queue so that we know what we've queued 1880 * to the other thread. 1881 */ 1882 STAILQ_FOREACH_SAFE(buf1, &local_queue, work_links, buf2) { 1883 if (buf1->buf_type != CAMDD_BUF_DATA) { 1884 errx(1, "%s: should have a data buffer, not an " 1885 "indirect buffer", __func__); 1886 } 1887 data = &buf1->buf_type_spec.data; 1888 1889 /* 1890 * We only need to send one EOF to the writer, and don't 1891 * need to continue sending EOFs after that. 1892 */ 1893 if (buf1->status == CAMDD_STATUS_EOF) { 1894 if (dev->flags & CAMDD_DEV_FLAG_EOF_SENT) { 1895 STAILQ_REMOVE(&local_queue, buf1, camdd_buf, 1896 work_links); 1897 camdd_release_buf(buf1); 1898 retval = 1; 1899 continue; 1900 } 1901 dev->flags |= CAMDD_DEV_FLAG_EOF_SENT; 1902 } 1903 1904 1905 STAILQ_INSERT_TAIL(&dev->peer_work_queue, buf1, links); 1906 peer_bytes_queued += (data->fill_len - data->resid); 1907 dev->peer_bytes_queued += (data->fill_len - data->resid); 1908 dev->num_peer_work_queue++; 1909 } 1910 1911 if (STAILQ_FIRST(&local_queue) == NULL) 1912 goto bailout; 1913 1914 /* 1915 * Drop our mutex and pick up the other thread's mutex. We need to 1916 * do this to avoid deadlocks. 1917 */ 1918 pthread_mutex_unlock(&dev->mutex); 1919 pthread_mutex_lock(&dev->peer_dev->mutex); 1920 1921 if (dev->peer_dev->flags & CAMDD_DEV_FLAG_ACTIVE) { 1922 /* 1923 * Put the buffers on the other thread's incoming work queue. 1924 */ 1925 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL; 1926 buf1 = STAILQ_FIRST(&local_queue)) { 1927 STAILQ_REMOVE_HEAD(&local_queue, work_links); 1928 STAILQ_INSERT_TAIL(&dev->peer_dev->work_queue, buf1, 1929 work_links); 1930 } 1931 /* 1932 * Send an event to the other thread's kqueue to let it know 1933 * that there is something on the work queue. 1934 */ 1935 retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL); 1936 if (retval == -1) 1937 warn("%s: unable to add peer work_queue kevent", 1938 __func__); 1939 else 1940 retval = 0; 1941 } else 1942 active = 0; 1943 1944 pthread_mutex_unlock(&dev->peer_dev->mutex); 1945 pthread_mutex_lock(&dev->mutex); 1946 1947 /* 1948 * If the other side isn't active, run through the queue and 1949 * release all of the buffers. 1950 */ 1951 if (active == 0) { 1952 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL; 1953 buf1 = STAILQ_FIRST(&local_queue)) { 1954 STAILQ_REMOVE_HEAD(&local_queue, work_links); 1955 STAILQ_REMOVE(&dev->peer_work_queue, buf1, camdd_buf, 1956 links); 1957 dev->num_peer_work_queue--; 1958 camdd_release_buf(buf1); 1959 } 1960 dev->peer_bytes_queued -= peer_bytes_queued; 1961 retval = 1; 1962 } 1963 1964 bailout: 1965 return (retval); 1966 } 1967 1968 /* 1969 * Return a buffer to the reader thread when we have completed writing it. 1970 */ 1971 int 1972 camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf) 1973 { 1974 struct kevent ke; 1975 int retval = 0; 1976 1977 /* 1978 * Setup the event to let the other thread know that we have 1979 * completed a buffer. 1980 */ 1981 EV_SET(&ke, (uintptr_t)&dev->peer_dev->peer_done_queue, EVFILT_USER, 0, 1982 NOTE_TRIGGER, 0, NULL); 1983 1984 /* 1985 * Drop our lock and acquire the other thread's lock before 1986 * manipulating 1987 */ 1988 pthread_mutex_unlock(&dev->mutex); 1989 pthread_mutex_lock(&dev->peer_dev->mutex); 1990 1991 /* 1992 * Put the buffer on the reader thread's peer done queue now that 1993 * we have completed it. 1994 */ 1995 STAILQ_INSERT_TAIL(&dev->peer_dev->peer_done_queue, peer_buf, 1996 work_links); 1997 dev->peer_dev->num_peer_done_queue++; 1998 1999 /* 2000 * Send an event to the peer thread to let it know that we've added 2001 * something to its peer done queue. 2002 */ 2003 retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL); 2004 if (retval == -1) 2005 warn("%s: unable to add peer_done_queue kevent", __func__); 2006 else 2007 retval = 0; 2008 2009 /* 2010 * Drop the other thread's lock and reacquire ours. 2011 */ 2012 pthread_mutex_unlock(&dev->peer_dev->mutex); 2013 pthread_mutex_lock(&dev->mutex); 2014 2015 return (retval); 2016 } 2017 2018 /* 2019 * Free a buffer that was written out by the writer thread and returned to 2020 * the reader thread. 2021 */ 2022 void 2023 camdd_peer_done(struct camdd_buf *buf) 2024 { 2025 struct camdd_dev *dev; 2026 struct camdd_buf_data *data; 2027 2028 dev = buf->dev; 2029 if (buf->buf_type != CAMDD_BUF_DATA) { 2030 errx(1, "%s: should have a data buffer, not an " 2031 "indirect buffer", __func__); 2032 } 2033 2034 data = &buf->buf_type_spec.data; 2035 2036 STAILQ_REMOVE(&dev->peer_work_queue, buf, camdd_buf, links); 2037 dev->num_peer_work_queue--; 2038 dev->peer_bytes_queued -= (data->fill_len - data->resid); 2039 2040 if (buf->status == CAMDD_STATUS_EOF) 2041 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF; 2042 2043 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 2044 } 2045 2046 /* 2047 * Assumes caller holds the lock for this device. 2048 */ 2049 void 2050 camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf, 2051 int *error_count) 2052 { 2053 int retval = 0; 2054 2055 /* 2056 * If we're the reader, we need to send the completed I/O 2057 * to the writer. If we're the writer, we need to just 2058 * free up resources, or let the reader know if we've 2059 * encountered an error. 2060 */ 2061 if (dev->write_dev == 0) { 2062 retval = camdd_queue_peer_buf(dev, buf); 2063 if (retval != 0) 2064 (*error_count)++; 2065 } else { 2066 struct camdd_buf *tmp_buf, *next_buf; 2067 2068 STAILQ_FOREACH_SAFE(tmp_buf, &buf->src_list, src_links, 2069 next_buf) { 2070 struct camdd_buf *src_buf; 2071 struct camdd_buf_indirect *indirect; 2072 2073 STAILQ_REMOVE(&buf->src_list, tmp_buf, 2074 camdd_buf, src_links); 2075 2076 tmp_buf->status = buf->status; 2077 2078 if (tmp_buf->buf_type == CAMDD_BUF_DATA) { 2079 camdd_complete_peer_buf(dev, tmp_buf); 2080 continue; 2081 } 2082 2083 indirect = &tmp_buf->buf_type_spec.indirect; 2084 src_buf = indirect->src_buf; 2085 src_buf->refcount--; 2086 /* 2087 * XXX KDM we probably need to account for 2088 * exactly how many bytes we were able to 2089 * write. Allocate the residual to the 2090 * first N buffers? Or just track the 2091 * number of bytes written? Right now the reader 2092 * doesn't do anything with a residual. 2093 */ 2094 src_buf->status = buf->status; 2095 if (src_buf->refcount <= 0) 2096 camdd_complete_peer_buf(dev, src_buf); 2097 STAILQ_INSERT_TAIL(&dev->free_indirect_queue, 2098 tmp_buf, links); 2099 } 2100 2101 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 2102 } 2103 } 2104 2105 /* 2106 * Fetch all completed commands from the pass(4) device. 2107 * 2108 * Returns the number of commands received, or -1 if any of the commands 2109 * completed with an error. Returns 0 if no commands are available. 2110 */ 2111 int 2112 camdd_pass_fetch(struct camdd_dev *dev) 2113 { 2114 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass; 2115 union ccb ccb; 2116 int retval = 0, num_fetched = 0, error_count = 0; 2117 2118 pthread_mutex_unlock(&dev->mutex); 2119 /* 2120 * XXX KDM we don't distinguish between EFAULT and ENOENT. 2121 */ 2122 while ((retval = ioctl(pass_dev->dev->fd, CAMIOGET, &ccb)) != -1) { 2123 struct camdd_buf *buf; 2124 struct camdd_buf_data *data; 2125 cam_status ccb_status; 2126 union ccb *buf_ccb; 2127 2128 buf = ccb.ccb_h.ccb_buf; 2129 data = &buf->buf_type_spec.data; 2130 buf_ccb = &data->ccb; 2131 2132 num_fetched++; 2133 2134 /* 2135 * Copy the CCB back out so we get status, sense data, etc. 2136 */ 2137 bcopy(&ccb, buf_ccb, sizeof(ccb)); 2138 2139 pthread_mutex_lock(&dev->mutex); 2140 2141 /* 2142 * We're now done, so take this off the active queue. 2143 */ 2144 STAILQ_REMOVE(&dev->active_queue, buf, camdd_buf, links); 2145 dev->cur_active_io--; 2146 2147 ccb_status = ccb.ccb_h.status & CAM_STATUS_MASK; 2148 if (ccb_status != CAM_REQ_CMP) { 2149 cam_error_print(pass_dev->dev, &ccb, CAM_ESF_ALL, 2150 CAM_EPF_ALL, stderr); 2151 } 2152 2153 data->resid = ccb.csio.resid; 2154 dev->bytes_transferred += (ccb.csio.dxfer_len - ccb.csio.resid); 2155 2156 if (buf->status == CAMDD_STATUS_NONE) 2157 buf->status = camdd_ccb_status(&ccb); 2158 if (buf->status == CAMDD_STATUS_ERROR) 2159 error_count++; 2160 else if (buf->status == CAMDD_STATUS_EOF) { 2161 /* 2162 * Once we queue this buffer to our partner thread, 2163 * he will know that we've hit EOF. 2164 */ 2165 dev->flags |= CAMDD_DEV_FLAG_EOF; 2166 } 2167 2168 camdd_complete_buf(dev, buf, &error_count); 2169 2170 /* 2171 * Unlock in preparation for the ioctl call. 2172 */ 2173 pthread_mutex_unlock(&dev->mutex); 2174 } 2175 2176 pthread_mutex_lock(&dev->mutex); 2177 2178 if (error_count > 0) 2179 return (-1); 2180 else 2181 return (num_fetched); 2182 } 2183 2184 /* 2185 * Returns -1 for error, 0 for success/continue, and 1 for resource 2186 * shortage/stop processing. 2187 */ 2188 int 2189 camdd_file_run(struct camdd_dev *dev) 2190 { 2191 struct camdd_dev_file *file_dev = &dev->dev_spec.file; 2192 struct camdd_buf_data *data; 2193 struct camdd_buf *buf; 2194 off_t io_offset; 2195 int retval = 0, write_dev = dev->write_dev; 2196 int error_count = 0, no_resources = 0, double_buf_needed = 0; 2197 uint32_t num_sectors = 0, db_len = 0; 2198 2199 buf = STAILQ_FIRST(&dev->run_queue); 2200 if (buf == NULL) { 2201 no_resources = 1; 2202 goto bailout; 2203 } else if ((dev->write_dev == 0) 2204 && (dev->flags & (CAMDD_DEV_FLAG_EOF | 2205 CAMDD_DEV_FLAG_EOF_SENT))) { 2206 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links); 2207 dev->num_run_queue--; 2208 buf->status = CAMDD_STATUS_EOF; 2209 error_count++; 2210 goto bailout; 2211 } 2212 2213 /* 2214 * If we're writing, we need to go through the source buffer list 2215 * and create an S/G list. 2216 */ 2217 if (write_dev != 0) { 2218 retval = camdd_buf_sg_create(buf, /*iovec*/ 1, 2219 dev->sector_size, &num_sectors, &double_buf_needed); 2220 if (retval != 0) { 2221 no_resources = 1; 2222 goto bailout; 2223 } 2224 } 2225 2226 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links); 2227 dev->num_run_queue--; 2228 2229 data = &buf->buf_type_spec.data; 2230 2231 /* 2232 * pread(2) and pwrite(2) offsets are byte offsets. 2233 */ 2234 io_offset = buf->lba * dev->sector_size; 2235 2236 /* 2237 * Unlock the mutex while we read or write. 2238 */ 2239 pthread_mutex_unlock(&dev->mutex); 2240 2241 /* 2242 * Note that we don't need to double buffer if we're the reader 2243 * because in that case, we have allocated a single buffer of 2244 * sufficient size to do the read. This copy is necessary on 2245 * writes because if one of the components of the S/G list is not 2246 * a sector size multiple, the kernel will reject the write. This 2247 * is unfortunate but not surprising. So this will make sure that 2248 * we're using a single buffer that is a multiple of the sector size. 2249 */ 2250 if ((double_buf_needed != 0) 2251 && (data->sg_count > 1) 2252 && (write_dev != 0)) { 2253 uint32_t cur_offset; 2254 int i; 2255 2256 if (file_dev->tmp_buf == NULL) 2257 file_dev->tmp_buf = calloc(dev->blocksize, 1); 2258 if (file_dev->tmp_buf == NULL) { 2259 buf->status = CAMDD_STATUS_ERROR; 2260 error_count++; 2261 pthread_mutex_lock(&dev->mutex); 2262 goto bailout; 2263 } 2264 for (i = 0, cur_offset = 0; i < data->sg_count; i++) { 2265 bcopy(data->iovec[i].iov_base, 2266 &file_dev->tmp_buf[cur_offset], 2267 data->iovec[i].iov_len); 2268 cur_offset += data->iovec[i].iov_len; 2269 } 2270 db_len = cur_offset; 2271 } 2272 2273 if (file_dev->file_flags & CAMDD_FF_CAN_SEEK) { 2274 if (write_dev == 0) { 2275 /* 2276 * XXX KDM is there any way we would need a S/G 2277 * list here? 2278 */ 2279 retval = pread(file_dev->fd, data->buf, 2280 buf->len, io_offset); 2281 } else { 2282 if (double_buf_needed != 0) { 2283 retval = pwrite(file_dev->fd, file_dev->tmp_buf, 2284 db_len, io_offset); 2285 } else if (data->sg_count == 0) { 2286 retval = pwrite(file_dev->fd, data->buf, 2287 data->fill_len, io_offset); 2288 } else { 2289 retval = pwritev(file_dev->fd, data->iovec, 2290 data->sg_count, io_offset); 2291 } 2292 } 2293 } else { 2294 if (write_dev == 0) { 2295 /* 2296 * XXX KDM is there any way we would need a S/G 2297 * list here? 2298 */ 2299 retval = read(file_dev->fd, data->buf, buf->len); 2300 } else { 2301 if (double_buf_needed != 0) { 2302 retval = write(file_dev->fd, file_dev->tmp_buf, 2303 db_len); 2304 } else if (data->sg_count == 0) { 2305 retval = write(file_dev->fd, data->buf, 2306 data->fill_len); 2307 } else { 2308 retval = writev(file_dev->fd, data->iovec, 2309 data->sg_count); 2310 } 2311 } 2312 } 2313 2314 /* We're done, re-acquire the lock */ 2315 pthread_mutex_lock(&dev->mutex); 2316 2317 if (retval >= (ssize_t)data->fill_len) { 2318 /* 2319 * If the bytes transferred is more than the request size, 2320 * that indicates an overrun, which should only happen at 2321 * the end of a transfer if we have to round up to a sector 2322 * boundary. 2323 */ 2324 if (buf->status == CAMDD_STATUS_NONE) 2325 buf->status = CAMDD_STATUS_OK; 2326 data->resid = 0; 2327 dev->bytes_transferred += retval; 2328 } else if (retval == -1) { 2329 warn("Error %s %s", (write_dev) ? "writing to" : 2330 "reading from", file_dev->filename); 2331 2332 buf->status = CAMDD_STATUS_ERROR; 2333 data->resid = data->fill_len; 2334 error_count++; 2335 2336 if (dev->debug == 0) 2337 goto bailout; 2338 2339 if ((double_buf_needed != 0) 2340 && (write_dev != 0)) { 2341 fprintf(stderr, "%s: fd %d, DB buf %p, len %u lba %ju " 2342 "offset %ju\n", __func__, file_dev->fd, 2343 file_dev->tmp_buf, db_len, (uintmax_t)buf->lba, 2344 (uintmax_t)io_offset); 2345 } else if (data->sg_count == 0) { 2346 fprintf(stderr, "%s: fd %d, buf %p, len %u, lba %ju " 2347 "offset %ju\n", __func__, file_dev->fd, data->buf, 2348 data->fill_len, (uintmax_t)buf->lba, 2349 (uintmax_t)io_offset); 2350 } else { 2351 int i; 2352 2353 fprintf(stderr, "%s: fd %d, len %u, lba %ju " 2354 "offset %ju\n", __func__, file_dev->fd, 2355 data->fill_len, (uintmax_t)buf->lba, 2356 (uintmax_t)io_offset); 2357 2358 for (i = 0; i < data->sg_count; i++) { 2359 fprintf(stderr, "index %d ptr %p len %zu\n", 2360 i, data->iovec[i].iov_base, 2361 data->iovec[i].iov_len); 2362 } 2363 } 2364 } else if (retval == 0) { 2365 buf->status = CAMDD_STATUS_EOF; 2366 if (dev->debug != 0) 2367 printf("%s: got EOF from %s!\n", __func__, 2368 file_dev->filename); 2369 data->resid = data->fill_len; 2370 error_count++; 2371 } else if (retval < (ssize_t)data->fill_len) { 2372 if (buf->status == CAMDD_STATUS_NONE) 2373 buf->status = CAMDD_STATUS_SHORT_IO; 2374 data->resid = data->fill_len - retval; 2375 dev->bytes_transferred += retval; 2376 } 2377 2378 bailout: 2379 if (buf != NULL) { 2380 if (buf->status == CAMDD_STATUS_EOF) { 2381 struct camdd_buf *buf2; 2382 dev->flags |= CAMDD_DEV_FLAG_EOF; 2383 STAILQ_FOREACH(buf2, &dev->run_queue, links) 2384 buf2->status = CAMDD_STATUS_EOF; 2385 } 2386 2387 camdd_complete_buf(dev, buf, &error_count); 2388 } 2389 2390 if (error_count != 0) 2391 return (-1); 2392 else if (no_resources != 0) 2393 return (1); 2394 else 2395 return (0); 2396 } 2397 2398 /* 2399 * Execute one command from the run queue. Returns 0 for success, 1 for 2400 * stop processing, and -1 for error. 2401 */ 2402 int 2403 camdd_pass_run(struct camdd_dev *dev) 2404 { 2405 struct camdd_buf *buf = NULL; 2406 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass; 2407 struct camdd_buf_data *data; 2408 uint32_t num_blocks, sectors_used = 0; 2409 union ccb *ccb; 2410 int retval = 0, is_write = dev->write_dev; 2411 int double_buf_needed = 0; 2412 2413 buf = STAILQ_FIRST(&dev->run_queue); 2414 if (buf == NULL) { 2415 retval = 1; 2416 goto bailout; 2417 } 2418 2419 /* 2420 * If we're writing, we need to go through the source buffer list 2421 * and create an S/G list. 2422 */ 2423 if (is_write != 0) { 2424 retval = camdd_buf_sg_create(buf, /*iovec*/ 0,dev->sector_size, 2425 §ors_used, &double_buf_needed); 2426 if (retval != 0) { 2427 retval = -1; 2428 goto bailout; 2429 } 2430 } 2431 2432 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links); 2433 dev->num_run_queue--; 2434 2435 data = &buf->buf_type_spec.data; 2436 2437 ccb = &data->ccb; 2438 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->csio); 2439 2440 /* 2441 * In almost every case the number of blocks should be the device 2442 * block size. The exception may be at the end of an I/O stream 2443 * for a partial block or at the end of a device. 2444 */ 2445 if (is_write != 0) 2446 num_blocks = sectors_used; 2447 else 2448 num_blocks = data->fill_len / pass_dev->block_len; 2449 2450 scsi_read_write(&ccb->csio, 2451 /*retries*/ dev->retry_count, 2452 /*cbfcnp*/ NULL, 2453 /*tag_action*/ MSG_SIMPLE_Q_TAG, 2454 /*readop*/ (dev->write_dev == 0) ? SCSI_RW_READ : 2455 SCSI_RW_WRITE, 2456 /*byte2*/ 0, 2457 /*minimum_cmd_size*/ dev->min_cmd_size, 2458 /*lba*/ buf->lba, 2459 /*block_count*/ num_blocks, 2460 /*data_ptr*/ (data->sg_count != 0) ? 2461 (uint8_t *)data->segs : data->buf, 2462 /*dxfer_len*/ (num_blocks * pass_dev->block_len), 2463 /*sense_len*/ SSD_FULL_SIZE, 2464 /*timeout*/ dev->io_timeout); 2465 2466 /* Disable freezing the device queue */ 2467 ccb->ccb_h.flags |= CAM_DEV_QFRZDIS; 2468 2469 if (dev->retry_count != 0) 2470 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER; 2471 2472 if (data->sg_count != 0) { 2473 ccb->csio.sglist_cnt = data->sg_count; 2474 ccb->ccb_h.flags |= CAM_DATA_SG; 2475 } 2476 2477 /* 2478 * Store a pointer to the buffer in the CCB. The kernel will 2479 * restore this when we get it back, and we'll use it to identify 2480 * the buffer this CCB came from. 2481 */ 2482 ccb->ccb_h.ccb_buf = buf; 2483 2484 /* 2485 * Unlock our mutex in preparation for issuing the ioctl. 2486 */ 2487 pthread_mutex_unlock(&dev->mutex); 2488 /* 2489 * Queue the CCB to the pass(4) driver. 2490 */ 2491 if (ioctl(pass_dev->dev->fd, CAMIOQUEUE, ccb) == -1) { 2492 pthread_mutex_lock(&dev->mutex); 2493 2494 warn("%s: error sending CAMIOQUEUE ioctl to %s%u", __func__, 2495 pass_dev->dev->device_name, pass_dev->dev->dev_unit_num); 2496 warn("%s: CCB address is %p", __func__, ccb); 2497 retval = -1; 2498 2499 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 2500 } else { 2501 pthread_mutex_lock(&dev->mutex); 2502 2503 dev->cur_active_io++; 2504 STAILQ_INSERT_TAIL(&dev->active_queue, buf, links); 2505 } 2506 2507 bailout: 2508 return (retval); 2509 } 2510 2511 int 2512 camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len) 2513 { 2514 struct camdd_dev_pass *pass_dev; 2515 uint32_t num_blocks; 2516 int retval = 0; 2517 2518 pass_dev = &dev->dev_spec.pass; 2519 2520 *lba = dev->next_io_pos_bytes / dev->sector_size; 2521 *len = dev->blocksize; 2522 num_blocks = *len / dev->sector_size; 2523 2524 /* 2525 * If max_sector is 0, then we have no set limit. This can happen 2526 * if we're writing to a file in a filesystem, or reading from 2527 * something like /dev/zero. 2528 */ 2529 if ((dev->max_sector != 0) 2530 || (dev->sector_io_limit != 0)) { 2531 uint64_t max_sector; 2532 2533 if ((dev->max_sector != 0) 2534 && (dev->sector_io_limit != 0)) 2535 max_sector = min(dev->sector_io_limit, dev->max_sector); 2536 else if (dev->max_sector != 0) 2537 max_sector = dev->max_sector; 2538 else 2539 max_sector = dev->sector_io_limit; 2540 2541 2542 /* 2543 * Check to see whether we're starting off past the end of 2544 * the device. If so, we need to just send an EOF 2545 * notification to the writer. 2546 */ 2547 if (*lba > max_sector) { 2548 *len = 0; 2549 retval = 1; 2550 } else if (((*lba + num_blocks) > max_sector + 1) 2551 || ((*lba + num_blocks) < *lba)) { 2552 /* 2553 * If we get here (but pass the first check), we 2554 * can trim the request length down to go to the 2555 * end of the device. 2556 */ 2557 num_blocks = (max_sector + 1) - *lba; 2558 *len = num_blocks * dev->sector_size; 2559 retval = 1; 2560 } 2561 } 2562 2563 dev->next_io_pos_bytes += *len; 2564 2565 return (retval); 2566 } 2567 2568 /* 2569 * Returns 0 for success, 1 for EOF detected, and -1 for failure. 2570 */ 2571 int 2572 camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf) 2573 { 2574 struct camdd_buf *buf = NULL; 2575 struct camdd_buf_data *data; 2576 struct camdd_dev_pass *pass_dev; 2577 size_t new_len; 2578 struct camdd_buf_data *rb_data; 2579 int is_write = dev->write_dev; 2580 int eof_flush_needed = 0; 2581 int retval = 0; 2582 int error; 2583 2584 pass_dev = &dev->dev_spec.pass; 2585 2586 /* 2587 * If we've gotten EOF or our partner has, we should not continue 2588 * queueing I/O. If we're a writer, though, we should continue 2589 * to write any buffers that don't have EOF status. 2590 */ 2591 if ((dev->flags & CAMDD_DEV_FLAG_EOF) 2592 || ((dev->flags & CAMDD_DEV_FLAG_PEER_EOF) 2593 && (is_write == 0))) { 2594 /* 2595 * Tell the worker thread that we have seen EOF. 2596 */ 2597 retval = 1; 2598 2599 /* 2600 * If we're the writer, send the buffer back with EOF status. 2601 */ 2602 if (is_write) { 2603 read_buf->status = CAMDD_STATUS_EOF; 2604 2605 error = camdd_complete_peer_buf(dev, read_buf); 2606 } 2607 goto bailout; 2608 } 2609 2610 if (is_write == 0) { 2611 buf = camdd_get_buf(dev, CAMDD_BUF_DATA); 2612 if (buf == NULL) { 2613 retval = -1; 2614 goto bailout; 2615 } 2616 data = &buf->buf_type_spec.data; 2617 2618 retval = camdd_get_next_lba_len(dev, &buf->lba, &buf->len); 2619 if (retval != 0) { 2620 buf->status = CAMDD_STATUS_EOF; 2621 2622 if ((buf->len == 0) 2623 && ((dev->flags & (CAMDD_DEV_FLAG_EOF_SENT | 2624 CAMDD_DEV_FLAG_EOF_QUEUED)) != 0)) { 2625 camdd_release_buf(buf); 2626 goto bailout; 2627 } 2628 dev->flags |= CAMDD_DEV_FLAG_EOF_QUEUED; 2629 } 2630 2631 data->fill_len = buf->len; 2632 data->src_start_offset = buf->lba * dev->sector_size; 2633 2634 /* 2635 * Put this on the run queue. 2636 */ 2637 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2638 dev->num_run_queue++; 2639 2640 /* We're done. */ 2641 goto bailout; 2642 } 2643 2644 /* 2645 * Check for new EOF status from the reader. 2646 */ 2647 if ((read_buf->status == CAMDD_STATUS_EOF) 2648 || (read_buf->status == CAMDD_STATUS_ERROR)) { 2649 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF; 2650 if ((STAILQ_FIRST(&dev->pending_queue) == NULL) 2651 && (read_buf->len == 0)) { 2652 camdd_complete_peer_buf(dev, read_buf); 2653 retval = 1; 2654 goto bailout; 2655 } else 2656 eof_flush_needed = 1; 2657 } 2658 2659 /* 2660 * See if we have a buffer we're composing with pieces from our 2661 * partner thread. 2662 */ 2663 buf = STAILQ_FIRST(&dev->pending_queue); 2664 if (buf == NULL) { 2665 uint64_t lba; 2666 ssize_t len; 2667 2668 retval = camdd_get_next_lba_len(dev, &lba, &len); 2669 if (retval != 0) { 2670 read_buf->status = CAMDD_STATUS_EOF; 2671 2672 if (len == 0) { 2673 dev->flags |= CAMDD_DEV_FLAG_EOF; 2674 error = camdd_complete_peer_buf(dev, read_buf); 2675 goto bailout; 2676 } 2677 } 2678 2679 /* 2680 * If we don't have a pending buffer, we need to grab a new 2681 * one from the free list or allocate another one. 2682 */ 2683 buf = camdd_get_buf(dev, CAMDD_BUF_DATA); 2684 if (buf == NULL) { 2685 retval = 1; 2686 goto bailout; 2687 } 2688 2689 buf->lba = lba; 2690 buf->len = len; 2691 2692 STAILQ_INSERT_TAIL(&dev->pending_queue, buf, links); 2693 dev->num_pending_queue++; 2694 } 2695 2696 data = &buf->buf_type_spec.data; 2697 2698 rb_data = &read_buf->buf_type_spec.data; 2699 2700 if ((rb_data->src_start_offset != dev->next_peer_pos_bytes) 2701 && (dev->debug != 0)) { 2702 printf("%s: WARNING: reader offset %#jx != expected offset " 2703 "%#jx\n", __func__, (uintmax_t)rb_data->src_start_offset, 2704 (uintmax_t)dev->next_peer_pos_bytes); 2705 } 2706 dev->next_peer_pos_bytes = rb_data->src_start_offset + 2707 (rb_data->fill_len - rb_data->resid); 2708 2709 new_len = (rb_data->fill_len - rb_data->resid) + data->fill_len; 2710 if (new_len < buf->len) { 2711 /* 2712 * There are three cases here: 2713 * 1. We need more data to fill up a block, so we put 2714 * this I/O on the queue and wait for more I/O. 2715 * 2. We have a pending buffer in the queue that is 2716 * smaller than our blocksize, but we got an EOF. So we 2717 * need to go ahead and flush the write out. 2718 * 3. We got an error. 2719 */ 2720 2721 /* 2722 * Increment our fill length. 2723 */ 2724 data->fill_len += (rb_data->fill_len - rb_data->resid); 2725 2726 /* 2727 * Add the new read buffer to the list for writing. 2728 */ 2729 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links); 2730 2731 /* Increment the count */ 2732 buf->src_count++; 2733 2734 if (eof_flush_needed == 0) { 2735 /* 2736 * We need to exit, because we don't have enough 2737 * data yet. 2738 */ 2739 goto bailout; 2740 } else { 2741 /* 2742 * Take the buffer off of the pending queue. 2743 */ 2744 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, 2745 links); 2746 dev->num_pending_queue--; 2747 2748 /* 2749 * If we need an EOF flush, but there is no data 2750 * to flush, go ahead and return this buffer. 2751 */ 2752 if (data->fill_len == 0) { 2753 camdd_complete_buf(dev, buf, /*error_count*/0); 2754 retval = 1; 2755 goto bailout; 2756 } 2757 2758 /* 2759 * Put this on the next queue for execution. 2760 */ 2761 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2762 dev->num_run_queue++; 2763 } 2764 } else if (new_len == buf->len) { 2765 /* 2766 * We have enough data to completey fill one block, 2767 * so we're ready to issue the I/O. 2768 */ 2769 2770 /* 2771 * Take the buffer off of the pending queue. 2772 */ 2773 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, links); 2774 dev->num_pending_queue--; 2775 2776 /* 2777 * Add the new read buffer to the list for writing. 2778 */ 2779 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links); 2780 2781 /* Increment the count */ 2782 buf->src_count++; 2783 2784 /* 2785 * Increment our fill length. 2786 */ 2787 data->fill_len += (rb_data->fill_len - rb_data->resid); 2788 2789 /* 2790 * Put this on the next queue for execution. 2791 */ 2792 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2793 dev->num_run_queue++; 2794 } else { 2795 struct camdd_buf *idb; 2796 struct camdd_buf_indirect *indirect; 2797 uint32_t len_to_go, cur_offset; 2798 2799 2800 idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT); 2801 if (idb == NULL) { 2802 retval = 1; 2803 goto bailout; 2804 } 2805 indirect = &idb->buf_type_spec.indirect; 2806 indirect->src_buf = read_buf; 2807 read_buf->refcount++; 2808 indirect->offset = 0; 2809 indirect->start_ptr = rb_data->buf; 2810 /* 2811 * We've already established that there is more 2812 * data in read_buf than we have room for in our 2813 * current write request. So this particular chunk 2814 * of the request should just be the remainder 2815 * needed to fill up a block. 2816 */ 2817 indirect->len = buf->len - (data->fill_len - data->resid); 2818 2819 camdd_buf_add_child(buf, idb); 2820 2821 /* 2822 * This buffer is ready to execute, so we can take 2823 * it off the pending queue and put it on the run 2824 * queue. 2825 */ 2826 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, 2827 links); 2828 dev->num_pending_queue--; 2829 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2830 dev->num_run_queue++; 2831 2832 cur_offset = indirect->offset + indirect->len; 2833 2834 /* 2835 * The resulting I/O would be too large to fit in 2836 * one block. We need to split this I/O into 2837 * multiple pieces. Allocate as many buffers as needed. 2838 */ 2839 for (len_to_go = rb_data->fill_len - rb_data->resid - 2840 indirect->len; len_to_go > 0;) { 2841 struct camdd_buf *new_buf; 2842 struct camdd_buf_data *new_data; 2843 uint64_t lba; 2844 ssize_t len; 2845 2846 retval = camdd_get_next_lba_len(dev, &lba, &len); 2847 if ((retval != 0) 2848 && (len == 0)) { 2849 /* 2850 * The device has already been marked 2851 * as EOF, and there is no space left. 2852 */ 2853 goto bailout; 2854 } 2855 2856 new_buf = camdd_get_buf(dev, CAMDD_BUF_DATA); 2857 if (new_buf == NULL) { 2858 retval = 1; 2859 goto bailout; 2860 } 2861 2862 new_buf->lba = lba; 2863 new_buf->len = len; 2864 2865 idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT); 2866 if (idb == NULL) { 2867 retval = 1; 2868 goto bailout; 2869 } 2870 2871 indirect = &idb->buf_type_spec.indirect; 2872 2873 indirect->src_buf = read_buf; 2874 read_buf->refcount++; 2875 indirect->offset = cur_offset; 2876 indirect->start_ptr = rb_data->buf + cur_offset; 2877 indirect->len = min(len_to_go, new_buf->len); 2878 #if 0 2879 if (((indirect->len % dev->sector_size) != 0) 2880 || ((indirect->offset % dev->sector_size) != 0)) { 2881 warnx("offset %ju len %ju not aligned with " 2882 "sector size %u", indirect->offset, 2883 (uintmax_t)indirect->len, dev->sector_size); 2884 } 2885 #endif 2886 cur_offset += indirect->len; 2887 len_to_go -= indirect->len; 2888 2889 camdd_buf_add_child(new_buf, idb); 2890 2891 new_data = &new_buf->buf_type_spec.data; 2892 2893 if ((new_data->fill_len == new_buf->len) 2894 || (eof_flush_needed != 0)) { 2895 STAILQ_INSERT_TAIL(&dev->run_queue, 2896 new_buf, links); 2897 dev->num_run_queue++; 2898 } else if (new_data->fill_len < buf->len) { 2899 STAILQ_INSERT_TAIL(&dev->pending_queue, 2900 new_buf, links); 2901 dev->num_pending_queue++; 2902 } else { 2903 warnx("%s: too much data in new " 2904 "buffer!", __func__); 2905 retval = 1; 2906 goto bailout; 2907 } 2908 } 2909 } 2910 2911 bailout: 2912 return (retval); 2913 } 2914 2915 void 2916 camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth, 2917 uint32_t *peer_depth, uint32_t *our_bytes, uint32_t *peer_bytes) 2918 { 2919 *our_depth = dev->cur_active_io + dev->num_run_queue; 2920 if (dev->num_peer_work_queue > 2921 dev->num_peer_done_queue) 2922 *peer_depth = dev->num_peer_work_queue - 2923 dev->num_peer_done_queue; 2924 else 2925 *peer_depth = 0; 2926 *our_bytes = *our_depth * dev->blocksize; 2927 *peer_bytes = dev->peer_bytes_queued; 2928 } 2929 2930 void 2931 camdd_sig_handler(int sig) 2932 { 2933 if (sig == SIGINFO) 2934 need_status = 1; 2935 else { 2936 need_exit = 1; 2937 error_exit = 1; 2938 } 2939 2940 sem_post(&camdd_sem); 2941 } 2942 2943 void 2944 camdd_print_status(struct camdd_dev *camdd_dev, struct camdd_dev *other_dev, 2945 struct timespec *start_time) 2946 { 2947 struct timespec done_time; 2948 uint64_t total_ns; 2949 long double mb_sec, total_sec; 2950 int error = 0; 2951 2952 error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &done_time); 2953 if (error != 0) { 2954 warn("Unable to get done time"); 2955 return; 2956 } 2957 2958 timespecsub(&done_time, start_time); 2959 2960 total_ns = done_time.tv_nsec + (done_time.tv_sec * 1000000000); 2961 total_sec = total_ns; 2962 total_sec /= 1000000000; 2963 2964 fprintf(stderr, "%ju bytes %s %s\n%ju bytes %s %s\n" 2965 "%.4Lf seconds elapsed\n", 2966 (uintmax_t)camdd_dev->bytes_transferred, 2967 (camdd_dev->write_dev == 0) ? "read from" : "written to", 2968 camdd_dev->device_name, 2969 (uintmax_t)other_dev->bytes_transferred, 2970 (other_dev->write_dev == 0) ? "read from" : "written to", 2971 other_dev->device_name, total_sec); 2972 2973 mb_sec = min(other_dev->bytes_transferred,camdd_dev->bytes_transferred); 2974 mb_sec /= 1024 * 1024; 2975 mb_sec *= 1000000000; 2976 mb_sec /= total_ns; 2977 fprintf(stderr, "%.2Lf MB/sec\n", mb_sec); 2978 } 2979 2980 int 2981 camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts, uint64_t max_io, 2982 int retry_count, int timeout) 2983 { 2984 struct cam_device *new_cam_dev = NULL; 2985 struct camdd_dev *devs[2]; 2986 struct timespec start_time; 2987 pthread_t threads[2]; 2988 int unit = 0; 2989 int error = 0; 2990 int i; 2991 2992 if (num_io_opts != 2) { 2993 warnx("Must have one input and one output path"); 2994 error = 1; 2995 goto bailout; 2996 } 2997 2998 bzero(devs, sizeof(devs)); 2999 3000 for (i = 0; i < num_io_opts; i++) { 3001 switch (io_opts[i].dev_type) { 3002 case CAMDD_DEV_PASS: { 3003 if (isdigit(io_opts[i].dev_name[0])) { 3004 camdd_argmask new_arglist = CAMDD_ARG_NONE; 3005 int bus = 0, target = 0, lun = 0; 3006 int rv; 3007 3008 /* device specified as bus:target[:lun] */ 3009 rv = parse_btl(io_opts[i].dev_name, &bus, 3010 &target, &lun, &new_arglist); 3011 if (rv < 2) { 3012 warnx("numeric device specification " 3013 "must be either bus:target, or " 3014 "bus:target:lun"); 3015 error = 1; 3016 goto bailout; 3017 } 3018 /* default to 0 if lun was not specified */ 3019 if ((new_arglist & CAMDD_ARG_LUN) == 0) { 3020 lun = 0; 3021 new_arglist |= CAMDD_ARG_LUN; 3022 } 3023 new_cam_dev = cam_open_btl(bus, target, lun, 3024 O_RDWR, NULL); 3025 } else { 3026 char name[30]; 3027 3028 if (cam_get_device(io_opts[i].dev_name, name, 3029 sizeof name, &unit) == -1) { 3030 warnx("%s", cam_errbuf); 3031 error = 1; 3032 goto bailout; 3033 } 3034 new_cam_dev = cam_open_spec_device(name, unit, 3035 O_RDWR, NULL); 3036 } 3037 3038 if (new_cam_dev == NULL) { 3039 warnx("%s", cam_errbuf); 3040 error = 1; 3041 goto bailout; 3042 } 3043 3044 devs[i] = camdd_probe_pass(new_cam_dev, 3045 /*io_opts*/ &io_opts[i], 3046 CAMDD_ARG_ERR_RECOVER, 3047 /*probe_retry_count*/ 3, 3048 /*probe_timeout*/ 5000, 3049 /*io_retry_count*/ retry_count, 3050 /*io_timeout*/ timeout); 3051 if (devs[i] == NULL) { 3052 warn("Unable to probe device %s%u", 3053 new_cam_dev->device_name, 3054 new_cam_dev->dev_unit_num); 3055 error = 1; 3056 goto bailout; 3057 } 3058 break; 3059 } 3060 case CAMDD_DEV_FILE: { 3061 int fd = -1; 3062 3063 if (io_opts[i].dev_name[0] == '-') { 3064 if (io_opts[i].write_dev != 0) 3065 fd = STDOUT_FILENO; 3066 else 3067 fd = STDIN_FILENO; 3068 } else { 3069 if (io_opts[i].write_dev != 0) { 3070 fd = open(io_opts[i].dev_name, 3071 O_RDWR | O_CREAT, S_IWUSR |S_IRUSR); 3072 } else { 3073 fd = open(io_opts[i].dev_name, 3074 O_RDONLY); 3075 } 3076 } 3077 if (fd == -1) { 3078 warn("error opening file %s", 3079 io_opts[i].dev_name); 3080 error = 1; 3081 goto bailout; 3082 } 3083 3084 devs[i] = camdd_probe_file(fd, &io_opts[i], 3085 retry_count, timeout); 3086 if (devs[i] == NULL) { 3087 error = 1; 3088 goto bailout; 3089 } 3090 3091 break; 3092 } 3093 default: 3094 warnx("Unknown device type %d (%s)", 3095 io_opts[i].dev_type, io_opts[i].dev_name); 3096 error = 1; 3097 goto bailout; 3098 break; /*NOTREACHED */ 3099 } 3100 3101 devs[i]->write_dev = io_opts[i].write_dev; 3102 3103 devs[i]->start_offset_bytes = io_opts[i].offset; 3104 3105 if (max_io != 0) { 3106 devs[i]->sector_io_limit = 3107 (devs[i]->start_offset_bytes / 3108 devs[i]->sector_size) + 3109 (max_io / devs[i]->sector_size) - 1; 3110 } 3111 3112 devs[i]->next_io_pos_bytes = devs[i]->start_offset_bytes; 3113 devs[i]->next_completion_pos_bytes =devs[i]->start_offset_bytes; 3114 } 3115 3116 devs[0]->peer_dev = devs[1]; 3117 devs[1]->peer_dev = devs[0]; 3118 devs[0]->next_peer_pos_bytes = devs[0]->peer_dev->next_io_pos_bytes; 3119 devs[1]->next_peer_pos_bytes = devs[1]->peer_dev->next_io_pos_bytes; 3120 3121 sem_init(&camdd_sem, /*pshared*/ 0, 0); 3122 3123 signal(SIGINFO, camdd_sig_handler); 3124 signal(SIGINT, camdd_sig_handler); 3125 3126 error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &start_time); 3127 if (error != 0) { 3128 warn("Unable to get start time"); 3129 goto bailout; 3130 } 3131 3132 for (i = 0; i < num_io_opts; i++) { 3133 error = pthread_create(&threads[i], NULL, camdd_worker, 3134 (void *)devs[i]); 3135 if (error != 0) { 3136 warnc(error, "pthread_create() failed"); 3137 goto bailout; 3138 } 3139 } 3140 3141 for (;;) { 3142 if ((sem_wait(&camdd_sem) == -1) 3143 || (need_exit != 0)) { 3144 struct kevent ke; 3145 3146 for (i = 0; i < num_io_opts; i++) { 3147 EV_SET(&ke, (uintptr_t)&devs[i]->work_queue, 3148 EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL); 3149 3150 devs[i]->flags |= CAMDD_DEV_FLAG_EOF; 3151 3152 error = kevent(devs[i]->kq, &ke, 1, NULL, 0, 3153 NULL); 3154 if (error == -1) 3155 warn("%s: unable to wake up thread", 3156 __func__); 3157 error = 0; 3158 } 3159 break; 3160 } else if (need_status != 0) { 3161 camdd_print_status(devs[0], devs[1], &start_time); 3162 need_status = 0; 3163 } 3164 } 3165 for (i = 0; i < num_io_opts; i++) { 3166 pthread_join(threads[i], NULL); 3167 } 3168 3169 camdd_print_status(devs[0], devs[1], &start_time); 3170 3171 bailout: 3172 3173 for (i = 0; i < num_io_opts; i++) 3174 camdd_free_dev(devs[i]); 3175 3176 return (error + error_exit); 3177 } 3178 3179 void 3180 usage(void) 3181 { 3182 fprintf(stderr, 3183 "usage: camdd <-i|-o pass=pass0,bs=1M,offset=1M,depth=4>\n" 3184 " <-i|-o file=/tmp/file,bs=512K,offset=1M>\n" 3185 " <-i|-o file=/dev/da0,bs=512K,offset=1M>\n" 3186 " <-i|-o file=/dev/nsa0,bs=512K>\n" 3187 " [-C retry_count][-E][-m max_io_amt][-t timeout_secs][-v][-h]\n" 3188 "Option description\n" 3189 "-i <arg=val> Specify input device/file and parameters\n" 3190 "-o <arg=val> Specify output device/file and parameters\n" 3191 "Input and Output parameters\n" 3192 "pass=name Specify a pass(4) device like pass0 or /dev/pass0\n" 3193 "file=name Specify a file or device, /tmp/foo, /dev/da0, /dev/null\n" 3194 " or - for stdin/stdout\n" 3195 "bs=blocksize Specify blocksize in bytes, or using K, M, G, etc. suffix\n" 3196 "offset=len Specify starting offset in bytes or using K, M, G suffix\n" 3197 " NOTE: offset cannot be specified on tapes, pipes, stdin/out\n" 3198 "depth=N Specify a numeric queue depth. This only applies to pass(4)\n" 3199 "mcs=N Specify a minimum cmd size for pass(4) read/write commands\n" 3200 "Optional arguments\n" 3201 "-C retry_cnt Specify a retry count for pass(4) devices\n" 3202 "-E Enable CAM error recovery for pass(4) devices\n" 3203 "-m max_io Specify the maximum amount to be transferred in bytes or\n" 3204 " using K, G, M, etc. suffixes\n" 3205 "-t timeout Specify the I/O timeout to use with pass(4) devices\n" 3206 "-v Enable verbose error recovery\n" 3207 "-h Print this message\n"); 3208 } 3209 3210 3211 int 3212 camdd_parse_io_opts(char *args, int is_write, struct camdd_io_opts *io_opts) 3213 { 3214 char *tmpstr, *tmpstr2; 3215 char *orig_tmpstr = NULL; 3216 int retval = 0; 3217 3218 io_opts->write_dev = is_write; 3219 3220 tmpstr = strdup(args); 3221 if (tmpstr == NULL) { 3222 warn("strdup failed"); 3223 retval = 1; 3224 goto bailout; 3225 } 3226 orig_tmpstr = tmpstr; 3227 while ((tmpstr2 = strsep(&tmpstr, ",")) != NULL) { 3228 char *name, *value; 3229 3230 /* 3231 * If the user creates an empty parameter by putting in two 3232 * commas, skip over it and look for the next field. 3233 */ 3234 if (*tmpstr2 == '\0') 3235 continue; 3236 3237 name = strsep(&tmpstr2, "="); 3238 if (*name == '\0') { 3239 warnx("Got empty I/O parameter name"); 3240 retval = 1; 3241 goto bailout; 3242 } 3243 value = strsep(&tmpstr2, "="); 3244 if ((value == NULL) 3245 || (*value == '\0')) { 3246 warnx("Empty I/O parameter value for %s", name); 3247 retval = 1; 3248 goto bailout; 3249 } 3250 if (strncasecmp(name, "file", 4) == 0) { 3251 io_opts->dev_type = CAMDD_DEV_FILE; 3252 io_opts->dev_name = strdup(value); 3253 if (io_opts->dev_name == NULL) { 3254 warn("Error allocating memory"); 3255 retval = 1; 3256 goto bailout; 3257 } 3258 } else if (strncasecmp(name, "pass", 4) == 0) { 3259 io_opts->dev_type = CAMDD_DEV_PASS; 3260 io_opts->dev_name = strdup(value); 3261 if (io_opts->dev_name == NULL) { 3262 warn("Error allocating memory"); 3263 retval = 1; 3264 goto bailout; 3265 } 3266 } else if ((strncasecmp(name, "bs", 2) == 0) 3267 || (strncasecmp(name, "blocksize", 9) == 0)) { 3268 retval = expand_number(value, &io_opts->blocksize); 3269 if (retval == -1) { 3270 warn("expand_number(3) failed on %s=%s", name, 3271 value); 3272 retval = 1; 3273 goto bailout; 3274 } 3275 } else if (strncasecmp(name, "depth", 5) == 0) { 3276 char *endptr; 3277 3278 io_opts->queue_depth = strtoull(value, &endptr, 0); 3279 if (*endptr != '\0') { 3280 warnx("invalid queue depth %s", value); 3281 retval = 1; 3282 goto bailout; 3283 } 3284 } else if (strncasecmp(name, "mcs", 3) == 0) { 3285 char *endptr; 3286 3287 io_opts->min_cmd_size = strtol(value, &endptr, 0); 3288 if ((*endptr != '\0') 3289 || ((io_opts->min_cmd_size > 16) 3290 || (io_opts->min_cmd_size < 0))) { 3291 warnx("invalid minimum cmd size %s", value); 3292 retval = 1; 3293 goto bailout; 3294 } 3295 } else if (strncasecmp(name, "offset", 6) == 0) { 3296 retval = expand_number(value, &io_opts->offset); 3297 if (retval == -1) { 3298 warn("expand_number(3) failed on %s=%s", name, 3299 value); 3300 retval = 1; 3301 goto bailout; 3302 } 3303 } else if (strncasecmp(name, "debug", 5) == 0) { 3304 char *endptr; 3305 3306 io_opts->debug = strtoull(value, &endptr, 0); 3307 if (*endptr != '\0') { 3308 warnx("invalid debug level %s", value); 3309 retval = 1; 3310 goto bailout; 3311 } 3312 } else { 3313 warnx("Unrecognized parameter %s=%s", name, value); 3314 } 3315 } 3316 bailout: 3317 free(orig_tmpstr); 3318 3319 return (retval); 3320 } 3321 3322 int 3323 main(int argc, char **argv) 3324 { 3325 int c; 3326 camdd_argmask arglist = CAMDD_ARG_NONE; 3327 int timeout = 0, retry_count = 1; 3328 int error = 0; 3329 uint64_t max_io = 0; 3330 struct camdd_io_opts *opt_list = NULL; 3331 3332 if (argc == 1) { 3333 usage(); 3334 exit(1); 3335 } 3336 3337 opt_list = calloc(2, sizeof(struct camdd_io_opts)); 3338 if (opt_list == NULL) { 3339 warn("Unable to allocate option list"); 3340 error = 1; 3341 goto bailout; 3342 } 3343 3344 while ((c = getopt(argc, argv, "C:Ehi:m:o:t:v")) != -1){ 3345 switch (c) { 3346 case 'C': 3347 retry_count = strtol(optarg, NULL, 0); 3348 if (retry_count < 0) 3349 errx(1, "retry count %d is < 0", 3350 retry_count); 3351 arglist |= CAMDD_ARG_RETRIES; 3352 break; 3353 case 'E': 3354 arglist |= CAMDD_ARG_ERR_RECOVER; 3355 break; 3356 case 'i': 3357 case 'o': 3358 if (((c == 'i') 3359 && (opt_list[0].dev_type != CAMDD_DEV_NONE)) 3360 || ((c == 'o') 3361 && (opt_list[1].dev_type != CAMDD_DEV_NONE))) { 3362 errx(1, "Only one input and output path " 3363 "allowed"); 3364 } 3365 error = camdd_parse_io_opts(optarg, (c == 'o') ? 1 : 0, 3366 (c == 'o') ? &opt_list[1] : &opt_list[0]); 3367 if (error != 0) 3368 goto bailout; 3369 break; 3370 case 'm': 3371 error = expand_number(optarg, &max_io); 3372 if (error == -1) { 3373 warn("invalid maximum I/O amount %s", optarg); 3374 error = 1; 3375 goto bailout; 3376 } 3377 break; 3378 case 't': 3379 timeout = strtol(optarg, NULL, 0); 3380 if (timeout < 0) 3381 errx(1, "invalid timeout %d", timeout); 3382 /* Convert the timeout from seconds to ms */ 3383 timeout *= 1000; 3384 arglist |= CAMDD_ARG_TIMEOUT; 3385 break; 3386 case 'v': 3387 arglist |= CAMDD_ARG_VERBOSE; 3388 break; 3389 case 'h': 3390 default: 3391 usage(); 3392 exit(1); 3393 break; /*NOTREACHED*/ 3394 } 3395 } 3396 3397 if ((opt_list[0].dev_type == CAMDD_DEV_NONE) 3398 || (opt_list[1].dev_type == CAMDD_DEV_NONE)) 3399 errx(1, "Must specify both -i and -o"); 3400 3401 /* 3402 * Set the timeout if the user hasn't specified one. 3403 */ 3404 if (timeout == 0) 3405 timeout = CAMDD_PASS_RW_TIMEOUT; 3406 3407 error = camdd_rw(opt_list, 2, max_io, retry_count, timeout); 3408 3409 bailout: 3410 free(opt_list); 3411 3412 exit(error); 3413 } 3414