1 /* 2 * Copyright (C) 1999 Eric Youngdale 3 * Copyright (C) 2014 Christoph Hellwig 4 * 5 * SCSI queueing library. 6 * Initial versions: Eric Youngdale (eric@andante.org). 7 * Based upon conversations with large numbers 8 * of people at Linux Expo. 9 */ 10 11 #include <linux/bio.h> 12 #include <linux/bitops.h> 13 #include <linux/blkdev.h> 14 #include <linux/completion.h> 15 #include <linux/kernel.h> 16 #include <linux/export.h> 17 #include <linux/mempool.h> 18 #include <linux/slab.h> 19 #include <linux/init.h> 20 #include <linux/pci.h> 21 #include <linux/delay.h> 22 #include <linux/hardirq.h> 23 #include <linux/scatterlist.h> 24 #include <linux/blk-mq.h> 25 26 #include <scsi/scsi.h> 27 #include <scsi/scsi_cmnd.h> 28 #include <scsi/scsi_dbg.h> 29 #include <scsi/scsi_device.h> 30 #include <scsi/scsi_driver.h> 31 #include <scsi/scsi_eh.h> 32 #include <scsi/scsi_host.h> 33 34 #include <trace/events/scsi.h> 35 36 #include "scsi_priv.h" 37 #include "scsi_logging.h" 38 39 40 #define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools) 41 #define SG_MEMPOOL_SIZE 2 42 43 struct scsi_host_sg_pool { 44 size_t size; 45 char *name; 46 struct kmem_cache *slab; 47 mempool_t *pool; 48 }; 49 50 #define SP(x) { x, "sgpool-" __stringify(x) } 51 #if (SCSI_MAX_SG_SEGMENTS < 32) 52 #error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater) 53 #endif 54 static struct scsi_host_sg_pool scsi_sg_pools[] = { 55 SP(8), 56 SP(16), 57 #if (SCSI_MAX_SG_SEGMENTS > 32) 58 SP(32), 59 #if (SCSI_MAX_SG_SEGMENTS > 64) 60 SP(64), 61 #if (SCSI_MAX_SG_SEGMENTS > 128) 62 SP(128), 63 #if (SCSI_MAX_SG_SEGMENTS > 256) 64 #error SCSI_MAX_SG_SEGMENTS is too large (256 MAX) 65 #endif 66 #endif 67 #endif 68 #endif 69 SP(SCSI_MAX_SG_SEGMENTS) 70 }; 71 #undef SP 72 73 struct kmem_cache *scsi_sdb_cache; 74 75 /* 76 * When to reinvoke queueing after a resource shortage. It's 3 msecs to 77 * not change behaviour from the previous unplug mechanism, experimentation 78 * may prove this needs changing. 79 */ 80 #define SCSI_QUEUE_DELAY 3 81 82 static void 83 scsi_set_blocked(struct scsi_cmnd *cmd, int reason) 84 { 85 struct Scsi_Host *host = cmd->device->host; 86 struct scsi_device *device = cmd->device; 87 struct scsi_target *starget = scsi_target(device); 88 89 /* 90 * Set the appropriate busy bit for the device/host. 91 * 92 * If the host/device isn't busy, assume that something actually 93 * completed, and that we should be able to queue a command now. 94 * 95 * Note that the prior mid-layer assumption that any host could 96 * always queue at least one command is now broken. The mid-layer 97 * will implement a user specifiable stall (see 98 * scsi_host.max_host_blocked and scsi_device.max_device_blocked) 99 * if a command is requeued with no other commands outstanding 100 * either for the device or for the host. 101 */ 102 switch (reason) { 103 case SCSI_MLQUEUE_HOST_BUSY: 104 atomic_set(&host->host_blocked, host->max_host_blocked); 105 break; 106 case SCSI_MLQUEUE_DEVICE_BUSY: 107 case SCSI_MLQUEUE_EH_RETRY: 108 atomic_set(&device->device_blocked, 109 device->max_device_blocked); 110 break; 111 case SCSI_MLQUEUE_TARGET_BUSY: 112 atomic_set(&starget->target_blocked, 113 starget->max_target_blocked); 114 break; 115 } 116 } 117 118 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) 119 { 120 struct scsi_device *sdev = cmd->device; 121 struct request_queue *q = cmd->request->q; 122 123 blk_mq_requeue_request(cmd->request); 124 blk_mq_kick_requeue_list(q); 125 put_device(&sdev->sdev_gendev); 126 } 127 128 /** 129 * __scsi_queue_insert - private queue insertion 130 * @cmd: The SCSI command being requeued 131 * @reason: The reason for the requeue 132 * @unbusy: Whether the queue should be unbusied 133 * 134 * This is a private queue insertion. The public interface 135 * scsi_queue_insert() always assumes the queue should be unbusied 136 * because it's always called before the completion. This function is 137 * for a requeue after completion, which should only occur in this 138 * file. 139 */ 140 static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy) 141 { 142 struct scsi_device *device = cmd->device; 143 struct request_queue *q = device->request_queue; 144 unsigned long flags; 145 146 SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd, 147 "Inserting command %p into mlqueue\n", cmd)); 148 149 scsi_set_blocked(cmd, reason); 150 151 /* 152 * Decrement the counters, since these commands are no longer 153 * active on the host/device. 154 */ 155 if (unbusy) 156 scsi_device_unbusy(device); 157 158 /* 159 * Requeue this command. It will go before all other commands 160 * that are already in the queue. Schedule requeue work under 161 * lock such that the kblockd_schedule_work() call happens 162 * before blk_cleanup_queue() finishes. 163 */ 164 cmd->result = 0; 165 if (q->mq_ops) { 166 scsi_mq_requeue_cmd(cmd); 167 return; 168 } 169 spin_lock_irqsave(q->queue_lock, flags); 170 blk_requeue_request(q, cmd->request); 171 kblockd_schedule_work(&device->requeue_work); 172 spin_unlock_irqrestore(q->queue_lock, flags); 173 } 174 175 /* 176 * Function: scsi_queue_insert() 177 * 178 * Purpose: Insert a command in the midlevel queue. 179 * 180 * Arguments: cmd - command that we are adding to queue. 181 * reason - why we are inserting command to queue. 182 * 183 * Lock status: Assumed that lock is not held upon entry. 184 * 185 * Returns: Nothing. 186 * 187 * Notes: We do this for one of two cases. Either the host is busy 188 * and it cannot accept any more commands for the time being, 189 * or the device returned QUEUE_FULL and can accept no more 190 * commands. 191 * Notes: This could be called either from an interrupt context or a 192 * normal process context. 193 */ 194 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) 195 { 196 __scsi_queue_insert(cmd, reason, 1); 197 } 198 /** 199 * scsi_execute - insert request and wait for the result 200 * @sdev: scsi device 201 * @cmd: scsi command 202 * @data_direction: data direction 203 * @buffer: data buffer 204 * @bufflen: len of buffer 205 * @sense: optional sense buffer 206 * @timeout: request timeout in seconds 207 * @retries: number of times to retry request 208 * @flags: or into request flags; 209 * @resid: optional residual length 210 * 211 * returns the req->errors value which is the scsi_cmnd result 212 * field. 213 */ 214 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, 215 int data_direction, void *buffer, unsigned bufflen, 216 unsigned char *sense, int timeout, int retries, u64 flags, 217 int *resid) 218 { 219 struct request *req; 220 int write = (data_direction == DMA_TO_DEVICE); 221 int ret = DRIVER_ERROR << 24; 222 223 req = blk_get_request(sdev->request_queue, write, __GFP_WAIT); 224 if (IS_ERR(req)) 225 return ret; 226 blk_rq_set_block_pc(req); 227 228 if (bufflen && blk_rq_map_kern(sdev->request_queue, req, 229 buffer, bufflen, __GFP_WAIT)) 230 goto out; 231 232 req->cmd_len = COMMAND_SIZE(cmd[0]); 233 memcpy(req->cmd, cmd, req->cmd_len); 234 req->sense = sense; 235 req->sense_len = 0; 236 req->retries = retries; 237 req->timeout = timeout; 238 req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT; 239 240 /* 241 * head injection *required* here otherwise quiesce won't work 242 */ 243 blk_execute_rq(req->q, NULL, req, 1); 244 245 /* 246 * Some devices (USB mass-storage in particular) may transfer 247 * garbage data together with a residue indicating that the data 248 * is invalid. Prevent the garbage from being misinterpreted 249 * and prevent security leaks by zeroing out the excess data. 250 */ 251 if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen)) 252 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len); 253 254 if (resid) 255 *resid = req->resid_len; 256 ret = req->errors; 257 out: 258 blk_put_request(req); 259 260 return ret; 261 } 262 EXPORT_SYMBOL(scsi_execute); 263 264 int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd, 265 int data_direction, void *buffer, unsigned bufflen, 266 struct scsi_sense_hdr *sshdr, int timeout, int retries, 267 int *resid, u64 flags) 268 { 269 char *sense = NULL; 270 int result; 271 272 if (sshdr) { 273 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO); 274 if (!sense) 275 return DRIVER_ERROR << 24; 276 } 277 result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen, 278 sense, timeout, retries, flags, resid); 279 if (sshdr) 280 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr); 281 282 kfree(sense); 283 return result; 284 } 285 EXPORT_SYMBOL(scsi_execute_req_flags); 286 287 /* 288 * Function: scsi_init_cmd_errh() 289 * 290 * Purpose: Initialize cmd fields related to error handling. 291 * 292 * Arguments: cmd - command that is ready to be queued. 293 * 294 * Notes: This function has the job of initializing a number of 295 * fields related to error handling. Typically this will 296 * be called once for each command, as required. 297 */ 298 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd) 299 { 300 cmd->serial_number = 0; 301 scsi_set_resid(cmd, 0); 302 memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); 303 if (cmd->cmd_len == 0) 304 cmd->cmd_len = scsi_command_size(cmd->cmnd); 305 } 306 307 void scsi_device_unbusy(struct scsi_device *sdev) 308 { 309 struct Scsi_Host *shost = sdev->host; 310 struct scsi_target *starget = scsi_target(sdev); 311 unsigned long flags; 312 313 atomic_dec(&shost->host_busy); 314 if (starget->can_queue > 0) 315 atomic_dec(&starget->target_busy); 316 317 if (unlikely(scsi_host_in_recovery(shost) && 318 (shost->host_failed || shost->host_eh_scheduled))) { 319 spin_lock_irqsave(shost->host_lock, flags); 320 scsi_eh_wakeup(shost); 321 spin_unlock_irqrestore(shost->host_lock, flags); 322 } 323 324 atomic_dec(&sdev->device_busy); 325 } 326 327 static void scsi_kick_queue(struct request_queue *q) 328 { 329 if (q->mq_ops) 330 blk_mq_start_hw_queues(q); 331 else 332 blk_run_queue(q); 333 } 334 335 /* 336 * Called for single_lun devices on IO completion. Clear starget_sdev_user, 337 * and call blk_run_queue for all the scsi_devices on the target - 338 * including current_sdev first. 339 * 340 * Called with *no* scsi locks held. 341 */ 342 static void scsi_single_lun_run(struct scsi_device *current_sdev) 343 { 344 struct Scsi_Host *shost = current_sdev->host; 345 struct scsi_device *sdev, *tmp; 346 struct scsi_target *starget = scsi_target(current_sdev); 347 unsigned long flags; 348 349 spin_lock_irqsave(shost->host_lock, flags); 350 starget->starget_sdev_user = NULL; 351 spin_unlock_irqrestore(shost->host_lock, flags); 352 353 /* 354 * Call blk_run_queue for all LUNs on the target, starting with 355 * current_sdev. We race with others (to set starget_sdev_user), 356 * but in most cases, we will be first. Ideally, each LU on the 357 * target would get some limited time or requests on the target. 358 */ 359 scsi_kick_queue(current_sdev->request_queue); 360 361 spin_lock_irqsave(shost->host_lock, flags); 362 if (starget->starget_sdev_user) 363 goto out; 364 list_for_each_entry_safe(sdev, tmp, &starget->devices, 365 same_target_siblings) { 366 if (sdev == current_sdev) 367 continue; 368 if (scsi_device_get(sdev)) 369 continue; 370 371 spin_unlock_irqrestore(shost->host_lock, flags); 372 scsi_kick_queue(sdev->request_queue); 373 spin_lock_irqsave(shost->host_lock, flags); 374 375 scsi_device_put(sdev); 376 } 377 out: 378 spin_unlock_irqrestore(shost->host_lock, flags); 379 } 380 381 static inline bool scsi_device_is_busy(struct scsi_device *sdev) 382 { 383 if (atomic_read(&sdev->device_busy) >= sdev->queue_depth) 384 return true; 385 if (atomic_read(&sdev->device_blocked) > 0) 386 return true; 387 return false; 388 } 389 390 static inline bool scsi_target_is_busy(struct scsi_target *starget) 391 { 392 if (starget->can_queue > 0) { 393 if (atomic_read(&starget->target_busy) >= starget->can_queue) 394 return true; 395 if (atomic_read(&starget->target_blocked) > 0) 396 return true; 397 } 398 return false; 399 } 400 401 static inline bool scsi_host_is_busy(struct Scsi_Host *shost) 402 { 403 if (shost->can_queue > 0 && 404 atomic_read(&shost->host_busy) >= shost->can_queue) 405 return true; 406 if (atomic_read(&shost->host_blocked) > 0) 407 return true; 408 if (shost->host_self_blocked) 409 return true; 410 return false; 411 } 412 413 static void scsi_starved_list_run(struct Scsi_Host *shost) 414 { 415 LIST_HEAD(starved_list); 416 struct scsi_device *sdev; 417 unsigned long flags; 418 419 spin_lock_irqsave(shost->host_lock, flags); 420 list_splice_init(&shost->starved_list, &starved_list); 421 422 while (!list_empty(&starved_list)) { 423 struct request_queue *slq; 424 425 /* 426 * As long as shost is accepting commands and we have 427 * starved queues, call blk_run_queue. scsi_request_fn 428 * drops the queue_lock and can add us back to the 429 * starved_list. 430 * 431 * host_lock protects the starved_list and starved_entry. 432 * scsi_request_fn must get the host_lock before checking 433 * or modifying starved_list or starved_entry. 434 */ 435 if (scsi_host_is_busy(shost)) 436 break; 437 438 sdev = list_entry(starved_list.next, 439 struct scsi_device, starved_entry); 440 list_del_init(&sdev->starved_entry); 441 if (scsi_target_is_busy(scsi_target(sdev))) { 442 list_move_tail(&sdev->starved_entry, 443 &shost->starved_list); 444 continue; 445 } 446 447 /* 448 * Once we drop the host lock, a racing scsi_remove_device() 449 * call may remove the sdev from the starved list and destroy 450 * it and the queue. Mitigate by taking a reference to the 451 * queue and never touching the sdev again after we drop the 452 * host lock. Note: if __scsi_remove_device() invokes 453 * blk_cleanup_queue() before the queue is run from this 454 * function then blk_run_queue() will return immediately since 455 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING. 456 */ 457 slq = sdev->request_queue; 458 if (!blk_get_queue(slq)) 459 continue; 460 spin_unlock_irqrestore(shost->host_lock, flags); 461 462 scsi_kick_queue(slq); 463 blk_put_queue(slq); 464 465 spin_lock_irqsave(shost->host_lock, flags); 466 } 467 /* put any unprocessed entries back */ 468 list_splice(&starved_list, &shost->starved_list); 469 spin_unlock_irqrestore(shost->host_lock, flags); 470 } 471 472 /* 473 * Function: scsi_run_queue() 474 * 475 * Purpose: Select a proper request queue to serve next 476 * 477 * Arguments: q - last request's queue 478 * 479 * Returns: Nothing 480 * 481 * Notes: The previous command was completely finished, start 482 * a new one if possible. 483 */ 484 static void scsi_run_queue(struct request_queue *q) 485 { 486 struct scsi_device *sdev = q->queuedata; 487 488 if (scsi_target(sdev)->single_lun) 489 scsi_single_lun_run(sdev); 490 if (!list_empty(&sdev->host->starved_list)) 491 scsi_starved_list_run(sdev->host); 492 493 if (q->mq_ops) 494 blk_mq_start_stopped_hw_queues(q, false); 495 else 496 blk_run_queue(q); 497 } 498 499 void scsi_requeue_run_queue(struct work_struct *work) 500 { 501 struct scsi_device *sdev; 502 struct request_queue *q; 503 504 sdev = container_of(work, struct scsi_device, requeue_work); 505 q = sdev->request_queue; 506 scsi_run_queue(q); 507 } 508 509 /* 510 * Function: scsi_requeue_command() 511 * 512 * Purpose: Handle post-processing of completed commands. 513 * 514 * Arguments: q - queue to operate on 515 * cmd - command that may need to be requeued. 516 * 517 * Returns: Nothing 518 * 519 * Notes: After command completion, there may be blocks left 520 * over which weren't finished by the previous command 521 * this can be for a number of reasons - the main one is 522 * I/O errors in the middle of the request, in which case 523 * we need to request the blocks that come after the bad 524 * sector. 525 * Notes: Upon return, cmd is a stale pointer. 526 */ 527 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd) 528 { 529 struct scsi_device *sdev = cmd->device; 530 struct request *req = cmd->request; 531 unsigned long flags; 532 533 spin_lock_irqsave(q->queue_lock, flags); 534 blk_unprep_request(req); 535 req->special = NULL; 536 scsi_put_command(cmd); 537 blk_requeue_request(q, req); 538 spin_unlock_irqrestore(q->queue_lock, flags); 539 540 scsi_run_queue(q); 541 542 put_device(&sdev->sdev_gendev); 543 } 544 545 void scsi_next_command(struct scsi_cmnd *cmd) 546 { 547 struct scsi_device *sdev = cmd->device; 548 struct request_queue *q = sdev->request_queue; 549 550 scsi_put_command(cmd); 551 scsi_run_queue(q); 552 553 put_device(&sdev->sdev_gendev); 554 } 555 556 void scsi_run_host_queues(struct Scsi_Host *shost) 557 { 558 struct scsi_device *sdev; 559 560 shost_for_each_device(sdev, shost) 561 scsi_run_queue(sdev->request_queue); 562 } 563 564 static inline unsigned int scsi_sgtable_index(unsigned short nents) 565 { 566 unsigned int index; 567 568 BUG_ON(nents > SCSI_MAX_SG_SEGMENTS); 569 570 if (nents <= 8) 571 index = 0; 572 else 573 index = get_count_order(nents) - 3; 574 575 return index; 576 } 577 578 static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents) 579 { 580 struct scsi_host_sg_pool *sgp; 581 582 sgp = scsi_sg_pools + scsi_sgtable_index(nents); 583 mempool_free(sgl, sgp->pool); 584 } 585 586 static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask) 587 { 588 struct scsi_host_sg_pool *sgp; 589 590 sgp = scsi_sg_pools + scsi_sgtable_index(nents); 591 return mempool_alloc(sgp->pool, gfp_mask); 592 } 593 594 static void scsi_free_sgtable(struct scsi_data_buffer *sdb, bool mq) 595 { 596 if (mq && sdb->table.nents <= SCSI_MAX_SG_SEGMENTS) 597 return; 598 __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, mq, scsi_sg_free); 599 } 600 601 static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, 602 gfp_t gfp_mask, bool mq) 603 { 604 struct scatterlist *first_chunk = NULL; 605 int ret; 606 607 BUG_ON(!nents); 608 609 if (mq) { 610 if (nents <= SCSI_MAX_SG_SEGMENTS) { 611 sdb->table.nents = nents; 612 sg_init_table(sdb->table.sgl, sdb->table.nents); 613 return 0; 614 } 615 first_chunk = sdb->table.sgl; 616 } 617 618 ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS, 619 first_chunk, gfp_mask, scsi_sg_alloc); 620 if (unlikely(ret)) 621 scsi_free_sgtable(sdb, mq); 622 return ret; 623 } 624 625 static void scsi_uninit_cmd(struct scsi_cmnd *cmd) 626 { 627 if (cmd->request->cmd_type == REQ_TYPE_FS) { 628 struct scsi_driver *drv = scsi_cmd_to_driver(cmd); 629 630 if (drv->uninit_command) 631 drv->uninit_command(cmd); 632 } 633 } 634 635 static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd) 636 { 637 if (cmd->sdb.table.nents) 638 scsi_free_sgtable(&cmd->sdb, true); 639 if (cmd->request->next_rq && cmd->request->next_rq->special) 640 scsi_free_sgtable(cmd->request->next_rq->special, true); 641 if (scsi_prot_sg_count(cmd)) 642 scsi_free_sgtable(cmd->prot_sdb, true); 643 } 644 645 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd) 646 { 647 struct scsi_device *sdev = cmd->device; 648 struct Scsi_Host *shost = sdev->host; 649 unsigned long flags; 650 651 scsi_mq_free_sgtables(cmd); 652 scsi_uninit_cmd(cmd); 653 654 if (shost->use_cmd_list) { 655 BUG_ON(list_empty(&cmd->list)); 656 spin_lock_irqsave(&sdev->list_lock, flags); 657 list_del_init(&cmd->list); 658 spin_unlock_irqrestore(&sdev->list_lock, flags); 659 } 660 } 661 662 /* 663 * Function: scsi_release_buffers() 664 * 665 * Purpose: Free resources allocate for a scsi_command. 666 * 667 * Arguments: cmd - command that we are bailing. 668 * 669 * Lock status: Assumed that no lock is held upon entry. 670 * 671 * Returns: Nothing 672 * 673 * Notes: In the event that an upper level driver rejects a 674 * command, we must release resources allocated during 675 * the __init_io() function. Primarily this would involve 676 * the scatter-gather table. 677 */ 678 static void scsi_release_buffers(struct scsi_cmnd *cmd) 679 { 680 if (cmd->sdb.table.nents) 681 scsi_free_sgtable(&cmd->sdb, false); 682 683 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 684 685 if (scsi_prot_sg_count(cmd)) 686 scsi_free_sgtable(cmd->prot_sdb, false); 687 } 688 689 static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd) 690 { 691 struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special; 692 693 scsi_free_sgtable(bidi_sdb, false); 694 kmem_cache_free(scsi_sdb_cache, bidi_sdb); 695 cmd->request->next_rq->special = NULL; 696 } 697 698 static bool scsi_end_request(struct request *req, int error, 699 unsigned int bytes, unsigned int bidi_bytes) 700 { 701 struct scsi_cmnd *cmd = req->special; 702 struct scsi_device *sdev = cmd->device; 703 struct request_queue *q = sdev->request_queue; 704 705 if (blk_update_request(req, error, bytes)) 706 return true; 707 708 /* Bidi request must be completed as a whole */ 709 if (unlikely(bidi_bytes) && 710 blk_update_request(req->next_rq, error, bidi_bytes)) 711 return true; 712 713 if (blk_queue_add_random(q)) 714 add_disk_randomness(req->rq_disk); 715 716 if (req->mq_ctx) { 717 /* 718 * In the MQ case the command gets freed by __blk_mq_end_request, 719 * so we have to do all cleanup that depends on it earlier. 720 * 721 * We also can't kick the queues from irq context, so we 722 * will have to defer it to a workqueue. 723 */ 724 scsi_mq_uninit_cmd(cmd); 725 726 __blk_mq_end_request(req, error); 727 728 if (scsi_target(sdev)->single_lun || 729 !list_empty(&sdev->host->starved_list)) 730 kblockd_schedule_work(&sdev->requeue_work); 731 else 732 blk_mq_start_stopped_hw_queues(q, true); 733 734 put_device(&sdev->sdev_gendev); 735 } else { 736 unsigned long flags; 737 738 if (bidi_bytes) 739 scsi_release_bidi_buffers(cmd); 740 741 spin_lock_irqsave(q->queue_lock, flags); 742 blk_finish_request(req, error); 743 spin_unlock_irqrestore(q->queue_lock, flags); 744 745 scsi_release_buffers(cmd); 746 scsi_next_command(cmd); 747 } 748 749 return false; 750 } 751 752 /** 753 * __scsi_error_from_host_byte - translate SCSI error code into errno 754 * @cmd: SCSI command (unused) 755 * @result: scsi error code 756 * 757 * Translate SCSI error code into standard UNIX errno. 758 * Return values: 759 * -ENOLINK temporary transport failure 760 * -EREMOTEIO permanent target failure, do not retry 761 * -EBADE permanent nexus failure, retry on other path 762 * -ENOSPC No write space available 763 * -ENODATA Medium error 764 * -EIO unspecified I/O error 765 */ 766 static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result) 767 { 768 int error = 0; 769 770 switch(host_byte(result)) { 771 case DID_TRANSPORT_FAILFAST: 772 error = -ENOLINK; 773 break; 774 case DID_TARGET_FAILURE: 775 set_host_byte(cmd, DID_OK); 776 error = -EREMOTEIO; 777 break; 778 case DID_NEXUS_FAILURE: 779 set_host_byte(cmd, DID_OK); 780 error = -EBADE; 781 break; 782 case DID_ALLOC_FAILURE: 783 set_host_byte(cmd, DID_OK); 784 error = -ENOSPC; 785 break; 786 case DID_MEDIUM_ERROR: 787 set_host_byte(cmd, DID_OK); 788 error = -ENODATA; 789 break; 790 default: 791 error = -EIO; 792 break; 793 } 794 795 return error; 796 } 797 798 /* 799 * Function: scsi_io_completion() 800 * 801 * Purpose: Completion processing for block device I/O requests. 802 * 803 * Arguments: cmd - command that is finished. 804 * 805 * Lock status: Assumed that no lock is held upon entry. 806 * 807 * Returns: Nothing 808 * 809 * Notes: We will finish off the specified number of sectors. If we 810 * are done, the command block will be released and the queue 811 * function will be goosed. If we are not done then we have to 812 * figure out what to do next: 813 * 814 * a) We can call scsi_requeue_command(). The request 815 * will be unprepared and put back on the queue. Then 816 * a new command will be created for it. This should 817 * be used if we made forward progress, or if we want 818 * to switch from READ(10) to READ(6) for example. 819 * 820 * b) We can call __scsi_queue_insert(). The request will 821 * be put back on the queue and retried using the same 822 * command as before, possibly after a delay. 823 * 824 * c) We can call scsi_end_request() with -EIO to fail 825 * the remainder of the request. 826 */ 827 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) 828 { 829 int result = cmd->result; 830 struct request_queue *q = cmd->device->request_queue; 831 struct request *req = cmd->request; 832 int error = 0; 833 struct scsi_sense_hdr sshdr; 834 int sense_valid = 0; 835 int sense_deferred = 0; 836 enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY, 837 ACTION_DELAYED_RETRY} action; 838 unsigned long wait_for = (cmd->allowed + 1) * req->timeout; 839 840 if (result) { 841 sense_valid = scsi_command_normalize_sense(cmd, &sshdr); 842 if (sense_valid) 843 sense_deferred = scsi_sense_is_deferred(&sshdr); 844 } 845 846 if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */ 847 if (result) { 848 if (sense_valid && req->sense) { 849 /* 850 * SG_IO wants current and deferred errors 851 */ 852 int len = 8 + cmd->sense_buffer[7]; 853 854 if (len > SCSI_SENSE_BUFFERSIZE) 855 len = SCSI_SENSE_BUFFERSIZE; 856 memcpy(req->sense, cmd->sense_buffer, len); 857 req->sense_len = len; 858 } 859 if (!sense_deferred) 860 error = __scsi_error_from_host_byte(cmd, result); 861 } 862 /* 863 * __scsi_error_from_host_byte may have reset the host_byte 864 */ 865 req->errors = cmd->result; 866 867 req->resid_len = scsi_get_resid(cmd); 868 869 if (scsi_bidi_cmnd(cmd)) { 870 /* 871 * Bidi commands Must be complete as a whole, 872 * both sides at once. 873 */ 874 req->next_rq->resid_len = scsi_in(cmd)->resid; 875 if (scsi_end_request(req, 0, blk_rq_bytes(req), 876 blk_rq_bytes(req->next_rq))) 877 BUG(); 878 return; 879 } 880 } else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) { 881 /* 882 * Certain non BLOCK_PC requests are commands that don't 883 * actually transfer anything (FLUSH), so cannot use 884 * good_bytes != blk_rq_bytes(req) as the signal for an error. 885 * This sets the error explicitly for the problem case. 886 */ 887 error = __scsi_error_from_host_byte(cmd, result); 888 } 889 890 /* no bidi support for !REQ_TYPE_BLOCK_PC yet */ 891 BUG_ON(blk_bidi_rq(req)); 892 893 /* 894 * Next deal with any sectors which we were able to correctly 895 * handle. 896 */ 897 SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd, 898 "%u sectors total, %d bytes done.\n", 899 blk_rq_sectors(req), good_bytes)); 900 901 /* 902 * Recovered errors need reporting, but they're always treated 903 * as success, so fiddle the result code here. For BLOCK_PC 904 * we already took a copy of the original into rq->errors which 905 * is what gets returned to the user 906 */ 907 if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) { 908 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip 909 * print since caller wants ATA registers. Only occurs on 910 * SCSI ATA PASS_THROUGH commands when CK_COND=1 911 */ 912 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d)) 913 ; 914 else if (!(req->cmd_flags & REQ_QUIET)) 915 scsi_print_sense("", cmd); 916 result = 0; 917 /* BLOCK_PC may have set error */ 918 error = 0; 919 } 920 921 /* 922 * If we finished all bytes in the request we are done now. 923 */ 924 if (!scsi_end_request(req, error, good_bytes, 0)) 925 return; 926 927 /* 928 * Kill remainder if no retrys. 929 */ 930 if (error && scsi_noretry_cmd(cmd)) { 931 if (scsi_end_request(req, error, blk_rq_bytes(req), 0)) 932 BUG(); 933 return; 934 } 935 936 /* 937 * If there had been no error, but we have leftover bytes in the 938 * requeues just queue the command up again. 939 */ 940 if (result == 0) 941 goto requeue; 942 943 error = __scsi_error_from_host_byte(cmd, result); 944 945 if (host_byte(result) == DID_RESET) { 946 /* Third party bus reset or reset for error recovery 947 * reasons. Just retry the command and see what 948 * happens. 949 */ 950 action = ACTION_RETRY; 951 } else if (sense_valid && !sense_deferred) { 952 switch (sshdr.sense_key) { 953 case UNIT_ATTENTION: 954 if (cmd->device->removable) { 955 /* Detected disc change. Set a bit 956 * and quietly refuse further access. 957 */ 958 cmd->device->changed = 1; 959 action = ACTION_FAIL; 960 } else { 961 /* Must have been a power glitch, or a 962 * bus reset. Could not have been a 963 * media change, so we just retry the 964 * command and see what happens. 965 */ 966 action = ACTION_RETRY; 967 } 968 break; 969 case ILLEGAL_REQUEST: 970 /* If we had an ILLEGAL REQUEST returned, then 971 * we may have performed an unsupported 972 * command. The only thing this should be 973 * would be a ten byte read where only a six 974 * byte read was supported. Also, on a system 975 * where READ CAPACITY failed, we may have 976 * read past the end of the disk. 977 */ 978 if ((cmd->device->use_10_for_rw && 979 sshdr.asc == 0x20 && sshdr.ascq == 0x00) && 980 (cmd->cmnd[0] == READ_10 || 981 cmd->cmnd[0] == WRITE_10)) { 982 /* This will issue a new 6-byte command. */ 983 cmd->device->use_10_for_rw = 0; 984 action = ACTION_REPREP; 985 } else if (sshdr.asc == 0x10) /* DIX */ { 986 action = ACTION_FAIL; 987 error = -EILSEQ; 988 /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */ 989 } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) { 990 action = ACTION_FAIL; 991 error = -EREMOTEIO; 992 } else 993 action = ACTION_FAIL; 994 break; 995 case ABORTED_COMMAND: 996 action = ACTION_FAIL; 997 if (sshdr.asc == 0x10) /* DIF */ 998 error = -EILSEQ; 999 break; 1000 case NOT_READY: 1001 /* If the device is in the process of becoming 1002 * ready, or has a temporary blockage, retry. 1003 */ 1004 if (sshdr.asc == 0x04) { 1005 switch (sshdr.ascq) { 1006 case 0x01: /* becoming ready */ 1007 case 0x04: /* format in progress */ 1008 case 0x05: /* rebuild in progress */ 1009 case 0x06: /* recalculation in progress */ 1010 case 0x07: /* operation in progress */ 1011 case 0x08: /* Long write in progress */ 1012 case 0x09: /* self test in progress */ 1013 case 0x14: /* space allocation in progress */ 1014 action = ACTION_DELAYED_RETRY; 1015 break; 1016 default: 1017 action = ACTION_FAIL; 1018 break; 1019 } 1020 } else 1021 action = ACTION_FAIL; 1022 break; 1023 case VOLUME_OVERFLOW: 1024 /* See SSC3rXX or current. */ 1025 action = ACTION_FAIL; 1026 break; 1027 default: 1028 action = ACTION_FAIL; 1029 break; 1030 } 1031 } else 1032 action = ACTION_FAIL; 1033 1034 if (action != ACTION_FAIL && 1035 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) 1036 action = ACTION_FAIL; 1037 1038 switch (action) { 1039 case ACTION_FAIL: 1040 /* Give up and fail the remainder of the request */ 1041 if (!(req->cmd_flags & REQ_QUIET)) { 1042 scsi_print_result(cmd); 1043 if (driver_byte(result) & DRIVER_SENSE) 1044 scsi_print_sense("", cmd); 1045 scsi_print_command(cmd); 1046 } 1047 if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0)) 1048 return; 1049 /*FALLTHRU*/ 1050 case ACTION_REPREP: 1051 requeue: 1052 /* Unprep the request and put it back at the head of the queue. 1053 * A new command will be prepared and issued. 1054 */ 1055 if (q->mq_ops) { 1056 cmd->request->cmd_flags &= ~REQ_DONTPREP; 1057 scsi_mq_uninit_cmd(cmd); 1058 scsi_mq_requeue_cmd(cmd); 1059 } else { 1060 scsi_release_buffers(cmd); 1061 scsi_requeue_command(q, cmd); 1062 } 1063 break; 1064 case ACTION_RETRY: 1065 /* Retry the same command immediately */ 1066 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0); 1067 break; 1068 case ACTION_DELAYED_RETRY: 1069 /* Retry the same command after a delay */ 1070 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0); 1071 break; 1072 } 1073 } 1074 1075 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb, 1076 gfp_t gfp_mask) 1077 { 1078 int count; 1079 1080 /* 1081 * If sg table allocation fails, requeue request later. 1082 */ 1083 if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments, 1084 gfp_mask, req->mq_ctx != NULL))) 1085 return BLKPREP_DEFER; 1086 1087 /* 1088 * Next, walk the list, and fill in the addresses and sizes of 1089 * each segment. 1090 */ 1091 count = blk_rq_map_sg(req->q, req, sdb->table.sgl); 1092 BUG_ON(count > sdb->table.nents); 1093 sdb->table.nents = count; 1094 sdb->length = blk_rq_bytes(req); 1095 return BLKPREP_OK; 1096 } 1097 1098 /* 1099 * Function: scsi_init_io() 1100 * 1101 * Purpose: SCSI I/O initialize function. 1102 * 1103 * Arguments: cmd - Command descriptor we wish to initialize 1104 * 1105 * Returns: 0 on success 1106 * BLKPREP_DEFER if the failure is retryable 1107 * BLKPREP_KILL if the failure is fatal 1108 */ 1109 int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask) 1110 { 1111 struct scsi_device *sdev = cmd->device; 1112 struct request *rq = cmd->request; 1113 bool is_mq = (rq->mq_ctx != NULL); 1114 int error; 1115 1116 BUG_ON(!rq->nr_phys_segments); 1117 1118 error = scsi_init_sgtable(rq, &cmd->sdb, gfp_mask); 1119 if (error) 1120 goto err_exit; 1121 1122 if (blk_bidi_rq(rq)) { 1123 if (!rq->q->mq_ops) { 1124 struct scsi_data_buffer *bidi_sdb = 1125 kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC); 1126 if (!bidi_sdb) { 1127 error = BLKPREP_DEFER; 1128 goto err_exit; 1129 } 1130 1131 rq->next_rq->special = bidi_sdb; 1132 } 1133 1134 error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special, 1135 GFP_ATOMIC); 1136 if (error) 1137 goto err_exit; 1138 } 1139 1140 if (blk_integrity_rq(rq)) { 1141 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb; 1142 int ivecs, count; 1143 1144 BUG_ON(prot_sdb == NULL); 1145 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio); 1146 1147 if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask, is_mq)) { 1148 error = BLKPREP_DEFER; 1149 goto err_exit; 1150 } 1151 1152 count = blk_rq_map_integrity_sg(rq->q, rq->bio, 1153 prot_sdb->table.sgl); 1154 BUG_ON(unlikely(count > ivecs)); 1155 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q))); 1156 1157 cmd->prot_sdb = prot_sdb; 1158 cmd->prot_sdb->table.nents = count; 1159 } 1160 1161 return BLKPREP_OK; 1162 err_exit: 1163 if (is_mq) { 1164 scsi_mq_free_sgtables(cmd); 1165 } else { 1166 scsi_release_buffers(cmd); 1167 cmd->request->special = NULL; 1168 scsi_put_command(cmd); 1169 put_device(&sdev->sdev_gendev); 1170 } 1171 return error; 1172 } 1173 EXPORT_SYMBOL(scsi_init_io); 1174 1175 static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev, 1176 struct request *req) 1177 { 1178 struct scsi_cmnd *cmd; 1179 1180 if (!req->special) { 1181 /* Bail if we can't get a reference to the device */ 1182 if (!get_device(&sdev->sdev_gendev)) 1183 return NULL; 1184 1185 cmd = scsi_get_command(sdev, GFP_ATOMIC); 1186 if (unlikely(!cmd)) { 1187 put_device(&sdev->sdev_gendev); 1188 return NULL; 1189 } 1190 req->special = cmd; 1191 } else { 1192 cmd = req->special; 1193 } 1194 1195 /* pull a tag out of the request if we have one */ 1196 cmd->tag = req->tag; 1197 cmd->request = req; 1198 1199 cmd->cmnd = req->cmd; 1200 cmd->prot_op = SCSI_PROT_NORMAL; 1201 1202 return cmd; 1203 } 1204 1205 static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req) 1206 { 1207 struct scsi_cmnd *cmd = req->special; 1208 1209 /* 1210 * BLOCK_PC requests may transfer data, in which case they must 1211 * a bio attached to them. Or they might contain a SCSI command 1212 * that does not transfer data, in which case they may optionally 1213 * submit a request without an attached bio. 1214 */ 1215 if (req->bio) { 1216 int ret = scsi_init_io(cmd, GFP_ATOMIC); 1217 if (unlikely(ret)) 1218 return ret; 1219 } else { 1220 BUG_ON(blk_rq_bytes(req)); 1221 1222 memset(&cmd->sdb, 0, sizeof(cmd->sdb)); 1223 } 1224 1225 cmd->cmd_len = req->cmd_len; 1226 cmd->transfersize = blk_rq_bytes(req); 1227 cmd->allowed = req->retries; 1228 return BLKPREP_OK; 1229 } 1230 1231 /* 1232 * Setup a REQ_TYPE_FS command. These are simple request from filesystems 1233 * that still need to be translated to SCSI CDBs from the ULD. 1234 */ 1235 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req) 1236 { 1237 struct scsi_cmnd *cmd = req->special; 1238 1239 if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh 1240 && sdev->scsi_dh_data->scsi_dh->prep_fn)) { 1241 int ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req); 1242 if (ret != BLKPREP_OK) 1243 return ret; 1244 } 1245 1246 memset(cmd->cmnd, 0, BLK_MAX_CDB); 1247 return scsi_cmd_to_driver(cmd)->init_command(cmd); 1248 } 1249 1250 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req) 1251 { 1252 struct scsi_cmnd *cmd = req->special; 1253 1254 if (!blk_rq_bytes(req)) 1255 cmd->sc_data_direction = DMA_NONE; 1256 else if (rq_data_dir(req) == WRITE) 1257 cmd->sc_data_direction = DMA_TO_DEVICE; 1258 else 1259 cmd->sc_data_direction = DMA_FROM_DEVICE; 1260 1261 switch (req->cmd_type) { 1262 case REQ_TYPE_FS: 1263 return scsi_setup_fs_cmnd(sdev, req); 1264 case REQ_TYPE_BLOCK_PC: 1265 return scsi_setup_blk_pc_cmnd(sdev, req); 1266 default: 1267 return BLKPREP_KILL; 1268 } 1269 } 1270 1271 static int 1272 scsi_prep_state_check(struct scsi_device *sdev, struct request *req) 1273 { 1274 int ret = BLKPREP_OK; 1275 1276 /* 1277 * If the device is not in running state we will reject some 1278 * or all commands. 1279 */ 1280 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1281 switch (sdev->sdev_state) { 1282 case SDEV_OFFLINE: 1283 case SDEV_TRANSPORT_OFFLINE: 1284 /* 1285 * If the device is offline we refuse to process any 1286 * commands. The device must be brought online 1287 * before trying any recovery commands. 1288 */ 1289 sdev_printk(KERN_ERR, sdev, 1290 "rejecting I/O to offline device\n"); 1291 ret = BLKPREP_KILL; 1292 break; 1293 case SDEV_DEL: 1294 /* 1295 * If the device is fully deleted, we refuse to 1296 * process any commands as well. 1297 */ 1298 sdev_printk(KERN_ERR, sdev, 1299 "rejecting I/O to dead device\n"); 1300 ret = BLKPREP_KILL; 1301 break; 1302 case SDEV_QUIESCE: 1303 case SDEV_BLOCK: 1304 case SDEV_CREATED_BLOCK: 1305 /* 1306 * If the devices is blocked we defer normal commands. 1307 */ 1308 if (!(req->cmd_flags & REQ_PREEMPT)) 1309 ret = BLKPREP_DEFER; 1310 break; 1311 default: 1312 /* 1313 * For any other not fully online state we only allow 1314 * special commands. In particular any user initiated 1315 * command is not allowed. 1316 */ 1317 if (!(req->cmd_flags & REQ_PREEMPT)) 1318 ret = BLKPREP_KILL; 1319 break; 1320 } 1321 } 1322 return ret; 1323 } 1324 1325 static int 1326 scsi_prep_return(struct request_queue *q, struct request *req, int ret) 1327 { 1328 struct scsi_device *sdev = q->queuedata; 1329 1330 switch (ret) { 1331 case BLKPREP_KILL: 1332 req->errors = DID_NO_CONNECT << 16; 1333 /* release the command and kill it */ 1334 if (req->special) { 1335 struct scsi_cmnd *cmd = req->special; 1336 scsi_release_buffers(cmd); 1337 scsi_put_command(cmd); 1338 put_device(&sdev->sdev_gendev); 1339 req->special = NULL; 1340 } 1341 break; 1342 case BLKPREP_DEFER: 1343 /* 1344 * If we defer, the blk_peek_request() returns NULL, but the 1345 * queue must be restarted, so we schedule a callback to happen 1346 * shortly. 1347 */ 1348 if (atomic_read(&sdev->device_busy) == 0) 1349 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1350 break; 1351 default: 1352 req->cmd_flags |= REQ_DONTPREP; 1353 } 1354 1355 return ret; 1356 } 1357 1358 static int scsi_prep_fn(struct request_queue *q, struct request *req) 1359 { 1360 struct scsi_device *sdev = q->queuedata; 1361 struct scsi_cmnd *cmd; 1362 int ret; 1363 1364 ret = scsi_prep_state_check(sdev, req); 1365 if (ret != BLKPREP_OK) 1366 goto out; 1367 1368 cmd = scsi_get_cmd_from_req(sdev, req); 1369 if (unlikely(!cmd)) { 1370 ret = BLKPREP_DEFER; 1371 goto out; 1372 } 1373 1374 ret = scsi_setup_cmnd(sdev, req); 1375 out: 1376 return scsi_prep_return(q, req, ret); 1377 } 1378 1379 static void scsi_unprep_fn(struct request_queue *q, struct request *req) 1380 { 1381 scsi_uninit_cmd(req->special); 1382 } 1383 1384 /* 1385 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else 1386 * return 0. 1387 * 1388 * Called with the queue_lock held. 1389 */ 1390 static inline int scsi_dev_queue_ready(struct request_queue *q, 1391 struct scsi_device *sdev) 1392 { 1393 unsigned int busy; 1394 1395 busy = atomic_inc_return(&sdev->device_busy) - 1; 1396 if (atomic_read(&sdev->device_blocked)) { 1397 if (busy) 1398 goto out_dec; 1399 1400 /* 1401 * unblock after device_blocked iterates to zero 1402 */ 1403 if (atomic_dec_return(&sdev->device_blocked) > 0) { 1404 /* 1405 * For the MQ case we take care of this in the caller. 1406 */ 1407 if (!q->mq_ops) 1408 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1409 goto out_dec; 1410 } 1411 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev, 1412 "unblocking device at zero depth\n")); 1413 } 1414 1415 if (busy >= sdev->queue_depth) 1416 goto out_dec; 1417 1418 return 1; 1419 out_dec: 1420 atomic_dec(&sdev->device_busy); 1421 return 0; 1422 } 1423 1424 /* 1425 * scsi_target_queue_ready: checks if there we can send commands to target 1426 * @sdev: scsi device on starget to check. 1427 */ 1428 static inline int scsi_target_queue_ready(struct Scsi_Host *shost, 1429 struct scsi_device *sdev) 1430 { 1431 struct scsi_target *starget = scsi_target(sdev); 1432 unsigned int busy; 1433 1434 if (starget->single_lun) { 1435 spin_lock_irq(shost->host_lock); 1436 if (starget->starget_sdev_user && 1437 starget->starget_sdev_user != sdev) { 1438 spin_unlock_irq(shost->host_lock); 1439 return 0; 1440 } 1441 starget->starget_sdev_user = sdev; 1442 spin_unlock_irq(shost->host_lock); 1443 } 1444 1445 if (starget->can_queue <= 0) 1446 return 1; 1447 1448 busy = atomic_inc_return(&starget->target_busy) - 1; 1449 if (atomic_read(&starget->target_blocked) > 0) { 1450 if (busy) 1451 goto starved; 1452 1453 /* 1454 * unblock after target_blocked iterates to zero 1455 */ 1456 if (atomic_dec_return(&starget->target_blocked) > 0) 1457 goto out_dec; 1458 1459 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget, 1460 "unblocking target at zero depth\n")); 1461 } 1462 1463 if (busy >= starget->can_queue) 1464 goto starved; 1465 1466 return 1; 1467 1468 starved: 1469 spin_lock_irq(shost->host_lock); 1470 list_move_tail(&sdev->starved_entry, &shost->starved_list); 1471 spin_unlock_irq(shost->host_lock); 1472 out_dec: 1473 if (starget->can_queue > 0) 1474 atomic_dec(&starget->target_busy); 1475 return 0; 1476 } 1477 1478 /* 1479 * scsi_host_queue_ready: if we can send requests to shost, return 1 else 1480 * return 0. We must end up running the queue again whenever 0 is 1481 * returned, else IO can hang. 1482 */ 1483 static inline int scsi_host_queue_ready(struct request_queue *q, 1484 struct Scsi_Host *shost, 1485 struct scsi_device *sdev) 1486 { 1487 unsigned int busy; 1488 1489 if (scsi_host_in_recovery(shost)) 1490 return 0; 1491 1492 busy = atomic_inc_return(&shost->host_busy) - 1; 1493 if (atomic_read(&shost->host_blocked) > 0) { 1494 if (busy) 1495 goto starved; 1496 1497 /* 1498 * unblock after host_blocked iterates to zero 1499 */ 1500 if (atomic_dec_return(&shost->host_blocked) > 0) 1501 goto out_dec; 1502 1503 SCSI_LOG_MLQUEUE(3, 1504 shost_printk(KERN_INFO, shost, 1505 "unblocking host at zero depth\n")); 1506 } 1507 1508 if (shost->can_queue > 0 && busy >= shost->can_queue) 1509 goto starved; 1510 if (shost->host_self_blocked) 1511 goto starved; 1512 1513 /* We're OK to process the command, so we can't be starved */ 1514 if (!list_empty(&sdev->starved_entry)) { 1515 spin_lock_irq(shost->host_lock); 1516 if (!list_empty(&sdev->starved_entry)) 1517 list_del_init(&sdev->starved_entry); 1518 spin_unlock_irq(shost->host_lock); 1519 } 1520 1521 return 1; 1522 1523 starved: 1524 spin_lock_irq(shost->host_lock); 1525 if (list_empty(&sdev->starved_entry)) 1526 list_add_tail(&sdev->starved_entry, &shost->starved_list); 1527 spin_unlock_irq(shost->host_lock); 1528 out_dec: 1529 atomic_dec(&shost->host_busy); 1530 return 0; 1531 } 1532 1533 /* 1534 * Busy state exporting function for request stacking drivers. 1535 * 1536 * For efficiency, no lock is taken to check the busy state of 1537 * shost/starget/sdev, since the returned value is not guaranteed and 1538 * may be changed after request stacking drivers call the function, 1539 * regardless of taking lock or not. 1540 * 1541 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi 1542 * needs to return 'not busy'. Otherwise, request stacking drivers 1543 * may hold requests forever. 1544 */ 1545 static int scsi_lld_busy(struct request_queue *q) 1546 { 1547 struct scsi_device *sdev = q->queuedata; 1548 struct Scsi_Host *shost; 1549 1550 if (blk_queue_dying(q)) 1551 return 0; 1552 1553 shost = sdev->host; 1554 1555 /* 1556 * Ignore host/starget busy state. 1557 * Since block layer does not have a concept of fairness across 1558 * multiple queues, congestion of host/starget needs to be handled 1559 * in SCSI layer. 1560 */ 1561 if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev)) 1562 return 1; 1563 1564 return 0; 1565 } 1566 1567 /* 1568 * Kill a request for a dead device 1569 */ 1570 static void scsi_kill_request(struct request *req, struct request_queue *q) 1571 { 1572 struct scsi_cmnd *cmd = req->special; 1573 struct scsi_device *sdev; 1574 struct scsi_target *starget; 1575 struct Scsi_Host *shost; 1576 1577 blk_start_request(req); 1578 1579 scmd_printk(KERN_INFO, cmd, "killing request\n"); 1580 1581 sdev = cmd->device; 1582 starget = scsi_target(sdev); 1583 shost = sdev->host; 1584 scsi_init_cmd_errh(cmd); 1585 cmd->result = DID_NO_CONNECT << 16; 1586 atomic_inc(&cmd->device->iorequest_cnt); 1587 1588 /* 1589 * SCSI request completion path will do scsi_device_unbusy(), 1590 * bump busy counts. To bump the counters, we need to dance 1591 * with the locks as normal issue path does. 1592 */ 1593 atomic_inc(&sdev->device_busy); 1594 atomic_inc(&shost->host_busy); 1595 if (starget->can_queue > 0) 1596 atomic_inc(&starget->target_busy); 1597 1598 blk_complete_request(req); 1599 } 1600 1601 static void scsi_softirq_done(struct request *rq) 1602 { 1603 struct scsi_cmnd *cmd = rq->special; 1604 unsigned long wait_for = (cmd->allowed + 1) * rq->timeout; 1605 int disposition; 1606 1607 INIT_LIST_HEAD(&cmd->eh_entry); 1608 1609 atomic_inc(&cmd->device->iodone_cnt); 1610 if (cmd->result) 1611 atomic_inc(&cmd->device->ioerr_cnt); 1612 1613 disposition = scsi_decide_disposition(cmd); 1614 if (disposition != SUCCESS && 1615 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) { 1616 sdev_printk(KERN_ERR, cmd->device, 1617 "timing out command, waited %lus\n", 1618 wait_for/HZ); 1619 disposition = SUCCESS; 1620 } 1621 1622 scsi_log_completion(cmd, disposition); 1623 1624 switch (disposition) { 1625 case SUCCESS: 1626 scsi_finish_command(cmd); 1627 break; 1628 case NEEDS_RETRY: 1629 scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY); 1630 break; 1631 case ADD_TO_MLQUEUE: 1632 scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY); 1633 break; 1634 default: 1635 if (!scsi_eh_scmd_add(cmd, 0)) 1636 scsi_finish_command(cmd); 1637 } 1638 } 1639 1640 /** 1641 * scsi_done - Invoke completion on finished SCSI command. 1642 * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives 1643 * ownership back to SCSI Core -- i.e. the LLDD has finished with it. 1644 * 1645 * Description: This function is the mid-level's (SCSI Core) interrupt routine, 1646 * which regains ownership of the SCSI command (de facto) from a LLDD, and 1647 * calls blk_complete_request() for further processing. 1648 * 1649 * This function is interrupt context safe. 1650 */ 1651 static void scsi_done(struct scsi_cmnd *cmd) 1652 { 1653 trace_scsi_dispatch_cmd_done(cmd); 1654 blk_complete_request(cmd->request); 1655 } 1656 1657 /* 1658 * Function: scsi_request_fn() 1659 * 1660 * Purpose: Main strategy routine for SCSI. 1661 * 1662 * Arguments: q - Pointer to actual queue. 1663 * 1664 * Returns: Nothing 1665 * 1666 * Lock status: IO request lock assumed to be held when called. 1667 */ 1668 static void scsi_request_fn(struct request_queue *q) 1669 __releases(q->queue_lock) 1670 __acquires(q->queue_lock) 1671 { 1672 struct scsi_device *sdev = q->queuedata; 1673 struct Scsi_Host *shost; 1674 struct scsi_cmnd *cmd; 1675 struct request *req; 1676 1677 /* 1678 * To start with, we keep looping until the queue is empty, or until 1679 * the host is no longer able to accept any more requests. 1680 */ 1681 shost = sdev->host; 1682 for (;;) { 1683 int rtn; 1684 /* 1685 * get next queueable request. We do this early to make sure 1686 * that the request is fully prepared even if we cannot 1687 * accept it. 1688 */ 1689 req = blk_peek_request(q); 1690 if (!req) 1691 break; 1692 1693 if (unlikely(!scsi_device_online(sdev))) { 1694 sdev_printk(KERN_ERR, sdev, 1695 "rejecting I/O to offline device\n"); 1696 scsi_kill_request(req, q); 1697 continue; 1698 } 1699 1700 if (!scsi_dev_queue_ready(q, sdev)) 1701 break; 1702 1703 /* 1704 * Remove the request from the request list. 1705 */ 1706 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req))) 1707 blk_start_request(req); 1708 1709 spin_unlock_irq(q->queue_lock); 1710 cmd = req->special; 1711 if (unlikely(cmd == NULL)) { 1712 printk(KERN_CRIT "impossible request in %s.\n" 1713 "please mail a stack trace to " 1714 "linux-scsi@vger.kernel.org\n", 1715 __func__); 1716 blk_dump_rq_flags(req, "foo"); 1717 BUG(); 1718 } 1719 1720 /* 1721 * We hit this when the driver is using a host wide 1722 * tag map. For device level tag maps the queue_depth check 1723 * in the device ready fn would prevent us from trying 1724 * to allocate a tag. Since the map is a shared host resource 1725 * we add the dev to the starved list so it eventually gets 1726 * a run when a tag is freed. 1727 */ 1728 if (blk_queue_tagged(q) && !blk_rq_tagged(req)) { 1729 spin_lock_irq(shost->host_lock); 1730 if (list_empty(&sdev->starved_entry)) 1731 list_add_tail(&sdev->starved_entry, 1732 &shost->starved_list); 1733 spin_unlock_irq(shost->host_lock); 1734 goto not_ready; 1735 } 1736 1737 if (!scsi_target_queue_ready(shost, sdev)) 1738 goto not_ready; 1739 1740 if (!scsi_host_queue_ready(q, shost, sdev)) 1741 goto host_not_ready; 1742 1743 /* 1744 * Finally, initialize any error handling parameters, and set up 1745 * the timers for timeouts. 1746 */ 1747 scsi_init_cmd_errh(cmd); 1748 1749 /* 1750 * Dispatch the command to the low-level driver. 1751 */ 1752 cmd->scsi_done = scsi_done; 1753 rtn = scsi_dispatch_cmd(cmd); 1754 if (rtn) { 1755 scsi_queue_insert(cmd, rtn); 1756 spin_lock_irq(q->queue_lock); 1757 goto out_delay; 1758 } 1759 spin_lock_irq(q->queue_lock); 1760 } 1761 1762 return; 1763 1764 host_not_ready: 1765 if (scsi_target(sdev)->can_queue > 0) 1766 atomic_dec(&scsi_target(sdev)->target_busy); 1767 not_ready: 1768 /* 1769 * lock q, handle tag, requeue req, and decrement device_busy. We 1770 * must return with queue_lock held. 1771 * 1772 * Decrementing device_busy without checking it is OK, as all such 1773 * cases (host limits or settings) should run the queue at some 1774 * later time. 1775 */ 1776 spin_lock_irq(q->queue_lock); 1777 blk_requeue_request(q, req); 1778 atomic_dec(&sdev->device_busy); 1779 out_delay: 1780 if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev)) 1781 blk_delay_queue(q, SCSI_QUEUE_DELAY); 1782 } 1783 1784 static inline int prep_to_mq(int ret) 1785 { 1786 switch (ret) { 1787 case BLKPREP_OK: 1788 return 0; 1789 case BLKPREP_DEFER: 1790 return BLK_MQ_RQ_QUEUE_BUSY; 1791 default: 1792 return BLK_MQ_RQ_QUEUE_ERROR; 1793 } 1794 } 1795 1796 static int scsi_mq_prep_fn(struct request *req) 1797 { 1798 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1799 struct scsi_device *sdev = req->q->queuedata; 1800 struct Scsi_Host *shost = sdev->host; 1801 unsigned char *sense_buf = cmd->sense_buffer; 1802 struct scatterlist *sg; 1803 1804 memset(cmd, 0, sizeof(struct scsi_cmnd)); 1805 1806 req->special = cmd; 1807 1808 cmd->request = req; 1809 cmd->device = sdev; 1810 cmd->sense_buffer = sense_buf; 1811 1812 cmd->tag = req->tag; 1813 1814 cmd->cmnd = req->cmd; 1815 cmd->prot_op = SCSI_PROT_NORMAL; 1816 1817 INIT_LIST_HEAD(&cmd->list); 1818 INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler); 1819 cmd->jiffies_at_alloc = jiffies; 1820 1821 if (shost->use_cmd_list) { 1822 spin_lock_irq(&sdev->list_lock); 1823 list_add_tail(&cmd->list, &sdev->cmd_list); 1824 spin_unlock_irq(&sdev->list_lock); 1825 } 1826 1827 sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size; 1828 cmd->sdb.table.sgl = sg; 1829 1830 if (scsi_host_get_prot(shost)) { 1831 cmd->prot_sdb = (void *)sg + 1832 shost->sg_tablesize * sizeof(struct scatterlist); 1833 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer)); 1834 1835 cmd->prot_sdb->table.sgl = 1836 (struct scatterlist *)(cmd->prot_sdb + 1); 1837 } 1838 1839 if (blk_bidi_rq(req)) { 1840 struct request *next_rq = req->next_rq; 1841 struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq); 1842 1843 memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer)); 1844 bidi_sdb->table.sgl = 1845 (struct scatterlist *)(bidi_sdb + 1); 1846 1847 next_rq->special = bidi_sdb; 1848 } 1849 1850 blk_mq_start_request(req); 1851 1852 return scsi_setup_cmnd(sdev, req); 1853 } 1854 1855 static void scsi_mq_done(struct scsi_cmnd *cmd) 1856 { 1857 trace_scsi_dispatch_cmd_done(cmd); 1858 blk_mq_complete_request(cmd->request); 1859 } 1860 1861 static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req, 1862 bool last) 1863 { 1864 struct request_queue *q = req->q; 1865 struct scsi_device *sdev = q->queuedata; 1866 struct Scsi_Host *shost = sdev->host; 1867 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 1868 int ret; 1869 int reason; 1870 1871 ret = prep_to_mq(scsi_prep_state_check(sdev, req)); 1872 if (ret) 1873 goto out; 1874 1875 ret = BLK_MQ_RQ_QUEUE_BUSY; 1876 if (!get_device(&sdev->sdev_gendev)) 1877 goto out; 1878 1879 if (!scsi_dev_queue_ready(q, sdev)) 1880 goto out_put_device; 1881 if (!scsi_target_queue_ready(shost, sdev)) 1882 goto out_dec_device_busy; 1883 if (!scsi_host_queue_ready(q, shost, sdev)) 1884 goto out_dec_target_busy; 1885 1886 1887 if (!(req->cmd_flags & REQ_DONTPREP)) { 1888 ret = prep_to_mq(scsi_mq_prep_fn(req)); 1889 if (ret) 1890 goto out_dec_host_busy; 1891 req->cmd_flags |= REQ_DONTPREP; 1892 } else { 1893 blk_mq_start_request(req); 1894 } 1895 1896 scsi_init_cmd_errh(cmd); 1897 cmd->scsi_done = scsi_mq_done; 1898 1899 reason = scsi_dispatch_cmd(cmd); 1900 if (reason) { 1901 scsi_set_blocked(cmd, reason); 1902 ret = BLK_MQ_RQ_QUEUE_BUSY; 1903 goto out_dec_host_busy; 1904 } 1905 1906 return BLK_MQ_RQ_QUEUE_OK; 1907 1908 out_dec_host_busy: 1909 atomic_dec(&shost->host_busy); 1910 out_dec_target_busy: 1911 if (scsi_target(sdev)->can_queue > 0) 1912 atomic_dec(&scsi_target(sdev)->target_busy); 1913 out_dec_device_busy: 1914 atomic_dec(&sdev->device_busy); 1915 out_put_device: 1916 put_device(&sdev->sdev_gendev); 1917 out: 1918 switch (ret) { 1919 case BLK_MQ_RQ_QUEUE_BUSY: 1920 blk_mq_stop_hw_queue(hctx); 1921 if (atomic_read(&sdev->device_busy) == 0 && 1922 !scsi_device_blocked(sdev)) 1923 blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY); 1924 break; 1925 case BLK_MQ_RQ_QUEUE_ERROR: 1926 /* 1927 * Make sure to release all allocated ressources when 1928 * we hit an error, as we will never see this command 1929 * again. 1930 */ 1931 if (req->cmd_flags & REQ_DONTPREP) 1932 scsi_mq_uninit_cmd(cmd); 1933 break; 1934 default: 1935 break; 1936 } 1937 return ret; 1938 } 1939 1940 static enum blk_eh_timer_return scsi_timeout(struct request *req, 1941 bool reserved) 1942 { 1943 if (reserved) 1944 return BLK_EH_RESET_TIMER; 1945 return scsi_times_out(req); 1946 } 1947 1948 static int scsi_init_request(void *data, struct request *rq, 1949 unsigned int hctx_idx, unsigned int request_idx, 1950 unsigned int numa_node) 1951 { 1952 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 1953 1954 cmd->sense_buffer = kzalloc_node(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL, 1955 numa_node); 1956 if (!cmd->sense_buffer) 1957 return -ENOMEM; 1958 return 0; 1959 } 1960 1961 static void scsi_exit_request(void *data, struct request *rq, 1962 unsigned int hctx_idx, unsigned int request_idx) 1963 { 1964 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 1965 1966 kfree(cmd->sense_buffer); 1967 } 1968 1969 static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost) 1970 { 1971 struct device *host_dev; 1972 u64 bounce_limit = 0xffffffff; 1973 1974 if (shost->unchecked_isa_dma) 1975 return BLK_BOUNCE_ISA; 1976 /* 1977 * Platforms with virtual-DMA translation 1978 * hardware have no practical limit. 1979 */ 1980 if (!PCI_DMA_BUS_IS_PHYS) 1981 return BLK_BOUNCE_ANY; 1982 1983 host_dev = scsi_get_device(shost); 1984 if (host_dev && host_dev->dma_mask) 1985 bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT; 1986 1987 return bounce_limit; 1988 } 1989 1990 static void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q) 1991 { 1992 struct device *dev = shost->dma_dev; 1993 1994 /* 1995 * this limit is imposed by hardware restrictions 1996 */ 1997 blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize, 1998 SCSI_MAX_SG_CHAIN_SEGMENTS)); 1999 2000 if (scsi_host_prot_dma(shost)) { 2001 shost->sg_prot_tablesize = 2002 min_not_zero(shost->sg_prot_tablesize, 2003 (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS); 2004 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize); 2005 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize); 2006 } 2007 2008 blk_queue_max_hw_sectors(q, shost->max_sectors); 2009 blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost)); 2010 blk_queue_segment_boundary(q, shost->dma_boundary); 2011 dma_set_seg_boundary(dev, shost->dma_boundary); 2012 2013 blk_queue_max_segment_size(q, dma_get_max_seg_size(dev)); 2014 2015 if (!shost->use_clustering) 2016 q->limits.cluster = 0; 2017 2018 /* 2019 * set a reasonable default alignment on word boundaries: the 2020 * host and device may alter it using 2021 * blk_queue_update_dma_alignment() later. 2022 */ 2023 blk_queue_dma_alignment(q, 0x03); 2024 } 2025 2026 struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost, 2027 request_fn_proc *request_fn) 2028 { 2029 struct request_queue *q; 2030 2031 q = blk_init_queue(request_fn, NULL); 2032 if (!q) 2033 return NULL; 2034 __scsi_init_queue(shost, q); 2035 return q; 2036 } 2037 EXPORT_SYMBOL(__scsi_alloc_queue); 2038 2039 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev) 2040 { 2041 struct request_queue *q; 2042 2043 q = __scsi_alloc_queue(sdev->host, scsi_request_fn); 2044 if (!q) 2045 return NULL; 2046 2047 blk_queue_prep_rq(q, scsi_prep_fn); 2048 blk_queue_unprep_rq(q, scsi_unprep_fn); 2049 blk_queue_softirq_done(q, scsi_softirq_done); 2050 blk_queue_rq_timed_out(q, scsi_times_out); 2051 blk_queue_lld_busy(q, scsi_lld_busy); 2052 return q; 2053 } 2054 2055 static struct blk_mq_ops scsi_mq_ops = { 2056 .map_queue = blk_mq_map_queue, 2057 .queue_rq = scsi_queue_rq, 2058 .complete = scsi_softirq_done, 2059 .timeout = scsi_timeout, 2060 .init_request = scsi_init_request, 2061 .exit_request = scsi_exit_request, 2062 }; 2063 2064 struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev) 2065 { 2066 sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set); 2067 if (IS_ERR(sdev->request_queue)) 2068 return NULL; 2069 2070 sdev->request_queue->queuedata = sdev; 2071 __scsi_init_queue(sdev->host, sdev->request_queue); 2072 return sdev->request_queue; 2073 } 2074 2075 int scsi_mq_setup_tags(struct Scsi_Host *shost) 2076 { 2077 unsigned int cmd_size, sgl_size, tbl_size; 2078 2079 tbl_size = shost->sg_tablesize; 2080 if (tbl_size > SCSI_MAX_SG_SEGMENTS) 2081 tbl_size = SCSI_MAX_SG_SEGMENTS; 2082 sgl_size = tbl_size * sizeof(struct scatterlist); 2083 cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size; 2084 if (scsi_host_get_prot(shost)) 2085 cmd_size += sizeof(struct scsi_data_buffer) + sgl_size; 2086 2087 memset(&shost->tag_set, 0, sizeof(shost->tag_set)); 2088 shost->tag_set.ops = &scsi_mq_ops; 2089 shost->tag_set.nr_hw_queues = 1; 2090 shost->tag_set.queue_depth = shost->can_queue; 2091 shost->tag_set.cmd_size = cmd_size; 2092 shost->tag_set.numa_node = NUMA_NO_NODE; 2093 shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2094 shost->tag_set.driver_data = shost; 2095 2096 return blk_mq_alloc_tag_set(&shost->tag_set); 2097 } 2098 2099 void scsi_mq_destroy_tags(struct Scsi_Host *shost) 2100 { 2101 blk_mq_free_tag_set(&shost->tag_set); 2102 } 2103 2104 /* 2105 * Function: scsi_block_requests() 2106 * 2107 * Purpose: Utility function used by low-level drivers to prevent further 2108 * commands from being queued to the device. 2109 * 2110 * Arguments: shost - Host in question 2111 * 2112 * Returns: Nothing 2113 * 2114 * Lock status: No locks are assumed held. 2115 * 2116 * Notes: There is no timer nor any other means by which the requests 2117 * get unblocked other than the low-level driver calling 2118 * scsi_unblock_requests(). 2119 */ 2120 void scsi_block_requests(struct Scsi_Host *shost) 2121 { 2122 shost->host_self_blocked = 1; 2123 } 2124 EXPORT_SYMBOL(scsi_block_requests); 2125 2126 /* 2127 * Function: scsi_unblock_requests() 2128 * 2129 * Purpose: Utility function used by low-level drivers to allow further 2130 * commands from being queued to the device. 2131 * 2132 * Arguments: shost - Host in question 2133 * 2134 * Returns: Nothing 2135 * 2136 * Lock status: No locks are assumed held. 2137 * 2138 * Notes: There is no timer nor any other means by which the requests 2139 * get unblocked other than the low-level driver calling 2140 * scsi_unblock_requests(). 2141 * 2142 * This is done as an API function so that changes to the 2143 * internals of the scsi mid-layer won't require wholesale 2144 * changes to drivers that use this feature. 2145 */ 2146 void scsi_unblock_requests(struct Scsi_Host *shost) 2147 { 2148 shost->host_self_blocked = 0; 2149 scsi_run_host_queues(shost); 2150 } 2151 EXPORT_SYMBOL(scsi_unblock_requests); 2152 2153 int __init scsi_init_queue(void) 2154 { 2155 int i; 2156 2157 scsi_sdb_cache = kmem_cache_create("scsi_data_buffer", 2158 sizeof(struct scsi_data_buffer), 2159 0, 0, NULL); 2160 if (!scsi_sdb_cache) { 2161 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n"); 2162 return -ENOMEM; 2163 } 2164 2165 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2166 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2167 int size = sgp->size * sizeof(struct scatterlist); 2168 2169 sgp->slab = kmem_cache_create(sgp->name, size, 0, 2170 SLAB_HWCACHE_ALIGN, NULL); 2171 if (!sgp->slab) { 2172 printk(KERN_ERR "SCSI: can't init sg slab %s\n", 2173 sgp->name); 2174 goto cleanup_sdb; 2175 } 2176 2177 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE, 2178 sgp->slab); 2179 if (!sgp->pool) { 2180 printk(KERN_ERR "SCSI: can't init sg mempool %s\n", 2181 sgp->name); 2182 goto cleanup_sdb; 2183 } 2184 } 2185 2186 return 0; 2187 2188 cleanup_sdb: 2189 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2190 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2191 if (sgp->pool) 2192 mempool_destroy(sgp->pool); 2193 if (sgp->slab) 2194 kmem_cache_destroy(sgp->slab); 2195 } 2196 kmem_cache_destroy(scsi_sdb_cache); 2197 2198 return -ENOMEM; 2199 } 2200 2201 void scsi_exit_queue(void) 2202 { 2203 int i; 2204 2205 kmem_cache_destroy(scsi_sdb_cache); 2206 2207 for (i = 0; i < SG_MEMPOOL_NR; i++) { 2208 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 2209 mempool_destroy(sgp->pool); 2210 kmem_cache_destroy(sgp->slab); 2211 } 2212 } 2213 2214 /** 2215 * scsi_mode_select - issue a mode select 2216 * @sdev: SCSI device to be queried 2217 * @pf: Page format bit (1 == standard, 0 == vendor specific) 2218 * @sp: Save page bit (0 == don't save, 1 == save) 2219 * @modepage: mode page being requested 2220 * @buffer: request buffer (may not be smaller than eight bytes) 2221 * @len: length of request buffer. 2222 * @timeout: command timeout 2223 * @retries: number of retries before failing 2224 * @data: returns a structure abstracting the mode header data 2225 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2226 * must be SCSI_SENSE_BUFFERSIZE big. 2227 * 2228 * Returns zero if successful; negative error number or scsi 2229 * status on error 2230 * 2231 */ 2232 int 2233 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage, 2234 unsigned char *buffer, int len, int timeout, int retries, 2235 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2236 { 2237 unsigned char cmd[10]; 2238 unsigned char *real_buffer; 2239 int ret; 2240 2241 memset(cmd, 0, sizeof(cmd)); 2242 cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0); 2243 2244 if (sdev->use_10_for_ms) { 2245 if (len > 65535) 2246 return -EINVAL; 2247 real_buffer = kmalloc(8 + len, GFP_KERNEL); 2248 if (!real_buffer) 2249 return -ENOMEM; 2250 memcpy(real_buffer + 8, buffer, len); 2251 len += 8; 2252 real_buffer[0] = 0; 2253 real_buffer[1] = 0; 2254 real_buffer[2] = data->medium_type; 2255 real_buffer[3] = data->device_specific; 2256 real_buffer[4] = data->longlba ? 0x01 : 0; 2257 real_buffer[5] = 0; 2258 real_buffer[6] = data->block_descriptor_length >> 8; 2259 real_buffer[7] = data->block_descriptor_length; 2260 2261 cmd[0] = MODE_SELECT_10; 2262 cmd[7] = len >> 8; 2263 cmd[8] = len; 2264 } else { 2265 if (len > 255 || data->block_descriptor_length > 255 || 2266 data->longlba) 2267 return -EINVAL; 2268 2269 real_buffer = kmalloc(4 + len, GFP_KERNEL); 2270 if (!real_buffer) 2271 return -ENOMEM; 2272 memcpy(real_buffer + 4, buffer, len); 2273 len += 4; 2274 real_buffer[0] = 0; 2275 real_buffer[1] = data->medium_type; 2276 real_buffer[2] = data->device_specific; 2277 real_buffer[3] = data->block_descriptor_length; 2278 2279 2280 cmd[0] = MODE_SELECT; 2281 cmd[4] = len; 2282 } 2283 2284 ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len, 2285 sshdr, timeout, retries, NULL); 2286 kfree(real_buffer); 2287 return ret; 2288 } 2289 EXPORT_SYMBOL_GPL(scsi_mode_select); 2290 2291 /** 2292 * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary. 2293 * @sdev: SCSI device to be queried 2294 * @dbd: set if mode sense will allow block descriptors to be returned 2295 * @modepage: mode page being requested 2296 * @buffer: request buffer (may not be smaller than eight bytes) 2297 * @len: length of request buffer. 2298 * @timeout: command timeout 2299 * @retries: number of retries before failing 2300 * @data: returns a structure abstracting the mode header data 2301 * @sshdr: place to put sense data (or NULL if no sense to be collected). 2302 * must be SCSI_SENSE_BUFFERSIZE big. 2303 * 2304 * Returns zero if unsuccessful, or the header offset (either 4 2305 * or 8 depending on whether a six or ten byte command was 2306 * issued) if successful. 2307 */ 2308 int 2309 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, 2310 unsigned char *buffer, int len, int timeout, int retries, 2311 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) 2312 { 2313 unsigned char cmd[12]; 2314 int use_10_for_ms; 2315 int header_length; 2316 int result; 2317 struct scsi_sense_hdr my_sshdr; 2318 2319 memset(data, 0, sizeof(*data)); 2320 memset(&cmd[0], 0, 12); 2321 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */ 2322 cmd[2] = modepage; 2323 2324 /* caller might not be interested in sense, but we need it */ 2325 if (!sshdr) 2326 sshdr = &my_sshdr; 2327 2328 retry: 2329 use_10_for_ms = sdev->use_10_for_ms; 2330 2331 if (use_10_for_ms) { 2332 if (len < 8) 2333 len = 8; 2334 2335 cmd[0] = MODE_SENSE_10; 2336 cmd[8] = len; 2337 header_length = 8; 2338 } else { 2339 if (len < 4) 2340 len = 4; 2341 2342 cmd[0] = MODE_SENSE; 2343 cmd[4] = len; 2344 header_length = 4; 2345 } 2346 2347 memset(buffer, 0, len); 2348 2349 result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len, 2350 sshdr, timeout, retries, NULL); 2351 2352 /* This code looks awful: what it's doing is making sure an 2353 * ILLEGAL REQUEST sense return identifies the actual command 2354 * byte as the problem. MODE_SENSE commands can return 2355 * ILLEGAL REQUEST if the code page isn't supported */ 2356 2357 if (use_10_for_ms && !scsi_status_is_good(result) && 2358 (driver_byte(result) & DRIVER_SENSE)) { 2359 if (scsi_sense_valid(sshdr)) { 2360 if ((sshdr->sense_key == ILLEGAL_REQUEST) && 2361 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) { 2362 /* 2363 * Invalid command operation code 2364 */ 2365 sdev->use_10_for_ms = 0; 2366 goto retry; 2367 } 2368 } 2369 } 2370 2371 if(scsi_status_is_good(result)) { 2372 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b && 2373 (modepage == 6 || modepage == 8))) { 2374 /* Initio breakage? */ 2375 header_length = 0; 2376 data->length = 13; 2377 data->medium_type = 0; 2378 data->device_specific = 0; 2379 data->longlba = 0; 2380 data->block_descriptor_length = 0; 2381 } else if(use_10_for_ms) { 2382 data->length = buffer[0]*256 + buffer[1] + 2; 2383 data->medium_type = buffer[2]; 2384 data->device_specific = buffer[3]; 2385 data->longlba = buffer[4] & 0x01; 2386 data->block_descriptor_length = buffer[6]*256 2387 + buffer[7]; 2388 } else { 2389 data->length = buffer[0] + 1; 2390 data->medium_type = buffer[1]; 2391 data->device_specific = buffer[2]; 2392 data->block_descriptor_length = buffer[3]; 2393 } 2394 data->header_length = header_length; 2395 } 2396 2397 return result; 2398 } 2399 EXPORT_SYMBOL(scsi_mode_sense); 2400 2401 /** 2402 * scsi_test_unit_ready - test if unit is ready 2403 * @sdev: scsi device to change the state of. 2404 * @timeout: command timeout 2405 * @retries: number of retries before failing 2406 * @sshdr_external: Optional pointer to struct scsi_sense_hdr for 2407 * returning sense. Make sure that this is cleared before passing 2408 * in. 2409 * 2410 * Returns zero if unsuccessful or an error if TUR failed. For 2411 * removable media, UNIT_ATTENTION sets ->changed flag. 2412 **/ 2413 int 2414 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries, 2415 struct scsi_sense_hdr *sshdr_external) 2416 { 2417 char cmd[] = { 2418 TEST_UNIT_READY, 0, 0, 0, 0, 0, 2419 }; 2420 struct scsi_sense_hdr *sshdr; 2421 int result; 2422 2423 if (!sshdr_external) 2424 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL); 2425 else 2426 sshdr = sshdr_external; 2427 2428 /* try to eat the UNIT_ATTENTION if there are enough retries */ 2429 do { 2430 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr, 2431 timeout, retries, NULL); 2432 if (sdev->removable && scsi_sense_valid(sshdr) && 2433 sshdr->sense_key == UNIT_ATTENTION) 2434 sdev->changed = 1; 2435 } while (scsi_sense_valid(sshdr) && 2436 sshdr->sense_key == UNIT_ATTENTION && --retries); 2437 2438 if (!sshdr_external) 2439 kfree(sshdr); 2440 return result; 2441 } 2442 EXPORT_SYMBOL(scsi_test_unit_ready); 2443 2444 /** 2445 * scsi_device_set_state - Take the given device through the device state model. 2446 * @sdev: scsi device to change the state of. 2447 * @state: state to change to. 2448 * 2449 * Returns zero if unsuccessful or an error if the requested 2450 * transition is illegal. 2451 */ 2452 int 2453 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state) 2454 { 2455 enum scsi_device_state oldstate = sdev->sdev_state; 2456 2457 if (state == oldstate) 2458 return 0; 2459 2460 switch (state) { 2461 case SDEV_CREATED: 2462 switch (oldstate) { 2463 case SDEV_CREATED_BLOCK: 2464 break; 2465 default: 2466 goto illegal; 2467 } 2468 break; 2469 2470 case SDEV_RUNNING: 2471 switch (oldstate) { 2472 case SDEV_CREATED: 2473 case SDEV_OFFLINE: 2474 case SDEV_TRANSPORT_OFFLINE: 2475 case SDEV_QUIESCE: 2476 case SDEV_BLOCK: 2477 break; 2478 default: 2479 goto illegal; 2480 } 2481 break; 2482 2483 case SDEV_QUIESCE: 2484 switch (oldstate) { 2485 case SDEV_RUNNING: 2486 case SDEV_OFFLINE: 2487 case SDEV_TRANSPORT_OFFLINE: 2488 break; 2489 default: 2490 goto illegal; 2491 } 2492 break; 2493 2494 case SDEV_OFFLINE: 2495 case SDEV_TRANSPORT_OFFLINE: 2496 switch (oldstate) { 2497 case SDEV_CREATED: 2498 case SDEV_RUNNING: 2499 case SDEV_QUIESCE: 2500 case SDEV_BLOCK: 2501 break; 2502 default: 2503 goto illegal; 2504 } 2505 break; 2506 2507 case SDEV_BLOCK: 2508 switch (oldstate) { 2509 case SDEV_RUNNING: 2510 case SDEV_CREATED_BLOCK: 2511 break; 2512 default: 2513 goto illegal; 2514 } 2515 break; 2516 2517 case SDEV_CREATED_BLOCK: 2518 switch (oldstate) { 2519 case SDEV_CREATED: 2520 break; 2521 default: 2522 goto illegal; 2523 } 2524 break; 2525 2526 case SDEV_CANCEL: 2527 switch (oldstate) { 2528 case SDEV_CREATED: 2529 case SDEV_RUNNING: 2530 case SDEV_QUIESCE: 2531 case SDEV_OFFLINE: 2532 case SDEV_TRANSPORT_OFFLINE: 2533 case SDEV_BLOCK: 2534 break; 2535 default: 2536 goto illegal; 2537 } 2538 break; 2539 2540 case SDEV_DEL: 2541 switch (oldstate) { 2542 case SDEV_CREATED: 2543 case SDEV_RUNNING: 2544 case SDEV_OFFLINE: 2545 case SDEV_TRANSPORT_OFFLINE: 2546 case SDEV_CANCEL: 2547 case SDEV_CREATED_BLOCK: 2548 break; 2549 default: 2550 goto illegal; 2551 } 2552 break; 2553 2554 } 2555 sdev->sdev_state = state; 2556 return 0; 2557 2558 illegal: 2559 SCSI_LOG_ERROR_RECOVERY(1, 2560 sdev_printk(KERN_ERR, sdev, 2561 "Illegal state transition %s->%s", 2562 scsi_device_state_name(oldstate), 2563 scsi_device_state_name(state)) 2564 ); 2565 return -EINVAL; 2566 } 2567 EXPORT_SYMBOL(scsi_device_set_state); 2568 2569 /** 2570 * sdev_evt_emit - emit a single SCSI device uevent 2571 * @sdev: associated SCSI device 2572 * @evt: event to emit 2573 * 2574 * Send a single uevent (scsi_event) to the associated scsi_device. 2575 */ 2576 static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt) 2577 { 2578 int idx = 0; 2579 char *envp[3]; 2580 2581 switch (evt->evt_type) { 2582 case SDEV_EVT_MEDIA_CHANGE: 2583 envp[idx++] = "SDEV_MEDIA_CHANGE=1"; 2584 break; 2585 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2586 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED"; 2587 break; 2588 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2589 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED"; 2590 break; 2591 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2592 envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED"; 2593 break; 2594 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2595 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED"; 2596 break; 2597 case SDEV_EVT_LUN_CHANGE_REPORTED: 2598 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED"; 2599 break; 2600 default: 2601 /* do nothing */ 2602 break; 2603 } 2604 2605 envp[idx++] = NULL; 2606 2607 kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp); 2608 } 2609 2610 /** 2611 * sdev_evt_thread - send a uevent for each scsi event 2612 * @work: work struct for scsi_device 2613 * 2614 * Dispatch queued events to their associated scsi_device kobjects 2615 * as uevents. 2616 */ 2617 void scsi_evt_thread(struct work_struct *work) 2618 { 2619 struct scsi_device *sdev; 2620 enum scsi_device_event evt_type; 2621 LIST_HEAD(event_list); 2622 2623 sdev = container_of(work, struct scsi_device, event_work); 2624 2625 for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++) 2626 if (test_and_clear_bit(evt_type, sdev->pending_events)) 2627 sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL); 2628 2629 while (1) { 2630 struct scsi_event *evt; 2631 struct list_head *this, *tmp; 2632 unsigned long flags; 2633 2634 spin_lock_irqsave(&sdev->list_lock, flags); 2635 list_splice_init(&sdev->event_list, &event_list); 2636 spin_unlock_irqrestore(&sdev->list_lock, flags); 2637 2638 if (list_empty(&event_list)) 2639 break; 2640 2641 list_for_each_safe(this, tmp, &event_list) { 2642 evt = list_entry(this, struct scsi_event, node); 2643 list_del(&evt->node); 2644 scsi_evt_emit(sdev, evt); 2645 kfree(evt); 2646 } 2647 } 2648 } 2649 2650 /** 2651 * sdev_evt_send - send asserted event to uevent thread 2652 * @sdev: scsi_device event occurred on 2653 * @evt: event to send 2654 * 2655 * Assert scsi device event asynchronously. 2656 */ 2657 void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt) 2658 { 2659 unsigned long flags; 2660 2661 #if 0 2662 /* FIXME: currently this check eliminates all media change events 2663 * for polled devices. Need to update to discriminate between AN 2664 * and polled events */ 2665 if (!test_bit(evt->evt_type, sdev->supported_events)) { 2666 kfree(evt); 2667 return; 2668 } 2669 #endif 2670 2671 spin_lock_irqsave(&sdev->list_lock, flags); 2672 list_add_tail(&evt->node, &sdev->event_list); 2673 schedule_work(&sdev->event_work); 2674 spin_unlock_irqrestore(&sdev->list_lock, flags); 2675 } 2676 EXPORT_SYMBOL_GPL(sdev_evt_send); 2677 2678 /** 2679 * sdev_evt_alloc - allocate a new scsi event 2680 * @evt_type: type of event to allocate 2681 * @gfpflags: GFP flags for allocation 2682 * 2683 * Allocates and returns a new scsi_event. 2684 */ 2685 struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type, 2686 gfp_t gfpflags) 2687 { 2688 struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags); 2689 if (!evt) 2690 return NULL; 2691 2692 evt->evt_type = evt_type; 2693 INIT_LIST_HEAD(&evt->node); 2694 2695 /* evt_type-specific initialization, if any */ 2696 switch (evt_type) { 2697 case SDEV_EVT_MEDIA_CHANGE: 2698 case SDEV_EVT_INQUIRY_CHANGE_REPORTED: 2699 case SDEV_EVT_CAPACITY_CHANGE_REPORTED: 2700 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED: 2701 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED: 2702 case SDEV_EVT_LUN_CHANGE_REPORTED: 2703 default: 2704 /* do nothing */ 2705 break; 2706 } 2707 2708 return evt; 2709 } 2710 EXPORT_SYMBOL_GPL(sdev_evt_alloc); 2711 2712 /** 2713 * sdev_evt_send_simple - send asserted event to uevent thread 2714 * @sdev: scsi_device event occurred on 2715 * @evt_type: type of event to send 2716 * @gfpflags: GFP flags for allocation 2717 * 2718 * Assert scsi device event asynchronously, given an event type. 2719 */ 2720 void sdev_evt_send_simple(struct scsi_device *sdev, 2721 enum scsi_device_event evt_type, gfp_t gfpflags) 2722 { 2723 struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags); 2724 if (!evt) { 2725 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n", 2726 evt_type); 2727 return; 2728 } 2729 2730 sdev_evt_send(sdev, evt); 2731 } 2732 EXPORT_SYMBOL_GPL(sdev_evt_send_simple); 2733 2734 /** 2735 * scsi_device_quiesce - Block user issued commands. 2736 * @sdev: scsi device to quiesce. 2737 * 2738 * This works by trying to transition to the SDEV_QUIESCE state 2739 * (which must be a legal transition). When the device is in this 2740 * state, only special requests will be accepted, all others will 2741 * be deferred. Since special requests may also be requeued requests, 2742 * a successful return doesn't guarantee the device will be 2743 * totally quiescent. 2744 * 2745 * Must be called with user context, may sleep. 2746 * 2747 * Returns zero if unsuccessful or an error if not. 2748 */ 2749 int 2750 scsi_device_quiesce(struct scsi_device *sdev) 2751 { 2752 int err = scsi_device_set_state(sdev, SDEV_QUIESCE); 2753 if (err) 2754 return err; 2755 2756 scsi_run_queue(sdev->request_queue); 2757 while (atomic_read(&sdev->device_busy)) { 2758 msleep_interruptible(200); 2759 scsi_run_queue(sdev->request_queue); 2760 } 2761 return 0; 2762 } 2763 EXPORT_SYMBOL(scsi_device_quiesce); 2764 2765 /** 2766 * scsi_device_resume - Restart user issued commands to a quiesced device. 2767 * @sdev: scsi device to resume. 2768 * 2769 * Moves the device from quiesced back to running and restarts the 2770 * queues. 2771 * 2772 * Must be called with user context, may sleep. 2773 */ 2774 void scsi_device_resume(struct scsi_device *sdev) 2775 { 2776 /* check if the device state was mutated prior to resume, and if 2777 * so assume the state is being managed elsewhere (for example 2778 * device deleted during suspend) 2779 */ 2780 if (sdev->sdev_state != SDEV_QUIESCE || 2781 scsi_device_set_state(sdev, SDEV_RUNNING)) 2782 return; 2783 scsi_run_queue(sdev->request_queue); 2784 } 2785 EXPORT_SYMBOL(scsi_device_resume); 2786 2787 static void 2788 device_quiesce_fn(struct scsi_device *sdev, void *data) 2789 { 2790 scsi_device_quiesce(sdev); 2791 } 2792 2793 void 2794 scsi_target_quiesce(struct scsi_target *starget) 2795 { 2796 starget_for_each_device(starget, NULL, device_quiesce_fn); 2797 } 2798 EXPORT_SYMBOL(scsi_target_quiesce); 2799 2800 static void 2801 device_resume_fn(struct scsi_device *sdev, void *data) 2802 { 2803 scsi_device_resume(sdev); 2804 } 2805 2806 void 2807 scsi_target_resume(struct scsi_target *starget) 2808 { 2809 starget_for_each_device(starget, NULL, device_resume_fn); 2810 } 2811 EXPORT_SYMBOL(scsi_target_resume); 2812 2813 /** 2814 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state 2815 * @sdev: device to block 2816 * 2817 * Block request made by scsi lld's to temporarily stop all 2818 * scsi commands on the specified device. Called from interrupt 2819 * or normal process context. 2820 * 2821 * Returns zero if successful or error if not 2822 * 2823 * Notes: 2824 * This routine transitions the device to the SDEV_BLOCK state 2825 * (which must be a legal transition). When the device is in this 2826 * state, all commands are deferred until the scsi lld reenables 2827 * the device with scsi_device_unblock or device_block_tmo fires. 2828 */ 2829 int 2830 scsi_internal_device_block(struct scsi_device *sdev) 2831 { 2832 struct request_queue *q = sdev->request_queue; 2833 unsigned long flags; 2834 int err = 0; 2835 2836 err = scsi_device_set_state(sdev, SDEV_BLOCK); 2837 if (err) { 2838 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK); 2839 2840 if (err) 2841 return err; 2842 } 2843 2844 /* 2845 * The device has transitioned to SDEV_BLOCK. Stop the 2846 * block layer from calling the midlayer with this device's 2847 * request queue. 2848 */ 2849 if (q->mq_ops) { 2850 blk_mq_stop_hw_queues(q); 2851 } else { 2852 spin_lock_irqsave(q->queue_lock, flags); 2853 blk_stop_queue(q); 2854 spin_unlock_irqrestore(q->queue_lock, flags); 2855 } 2856 2857 return 0; 2858 } 2859 EXPORT_SYMBOL_GPL(scsi_internal_device_block); 2860 2861 /** 2862 * scsi_internal_device_unblock - resume a device after a block request 2863 * @sdev: device to resume 2864 * @new_state: state to set devices to after unblocking 2865 * 2866 * Called by scsi lld's or the midlayer to restart the device queue 2867 * for the previously suspended scsi device. Called from interrupt or 2868 * normal process context. 2869 * 2870 * Returns zero if successful or error if not. 2871 * 2872 * Notes: 2873 * This routine transitions the device to the SDEV_RUNNING state 2874 * or to one of the offline states (which must be a legal transition) 2875 * allowing the midlayer to goose the queue for this device. 2876 */ 2877 int 2878 scsi_internal_device_unblock(struct scsi_device *sdev, 2879 enum scsi_device_state new_state) 2880 { 2881 struct request_queue *q = sdev->request_queue; 2882 unsigned long flags; 2883 2884 /* 2885 * Try to transition the scsi device to SDEV_RUNNING or one of the 2886 * offlined states and goose the device queue if successful. 2887 */ 2888 if ((sdev->sdev_state == SDEV_BLOCK) || 2889 (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE)) 2890 sdev->sdev_state = new_state; 2891 else if (sdev->sdev_state == SDEV_CREATED_BLOCK) { 2892 if (new_state == SDEV_TRANSPORT_OFFLINE || 2893 new_state == SDEV_OFFLINE) 2894 sdev->sdev_state = new_state; 2895 else 2896 sdev->sdev_state = SDEV_CREATED; 2897 } else if (sdev->sdev_state != SDEV_CANCEL && 2898 sdev->sdev_state != SDEV_OFFLINE) 2899 return -EINVAL; 2900 2901 if (q->mq_ops) { 2902 blk_mq_start_stopped_hw_queues(q, false); 2903 } else { 2904 spin_lock_irqsave(q->queue_lock, flags); 2905 blk_start_queue(q); 2906 spin_unlock_irqrestore(q->queue_lock, flags); 2907 } 2908 2909 return 0; 2910 } 2911 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock); 2912 2913 static void 2914 device_block(struct scsi_device *sdev, void *data) 2915 { 2916 scsi_internal_device_block(sdev); 2917 } 2918 2919 static int 2920 target_block(struct device *dev, void *data) 2921 { 2922 if (scsi_is_target_device(dev)) 2923 starget_for_each_device(to_scsi_target(dev), NULL, 2924 device_block); 2925 return 0; 2926 } 2927 2928 void 2929 scsi_target_block(struct device *dev) 2930 { 2931 if (scsi_is_target_device(dev)) 2932 starget_for_each_device(to_scsi_target(dev), NULL, 2933 device_block); 2934 else 2935 device_for_each_child(dev, NULL, target_block); 2936 } 2937 EXPORT_SYMBOL_GPL(scsi_target_block); 2938 2939 static void 2940 device_unblock(struct scsi_device *sdev, void *data) 2941 { 2942 scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data); 2943 } 2944 2945 static int 2946 target_unblock(struct device *dev, void *data) 2947 { 2948 if (scsi_is_target_device(dev)) 2949 starget_for_each_device(to_scsi_target(dev), data, 2950 device_unblock); 2951 return 0; 2952 } 2953 2954 void 2955 scsi_target_unblock(struct device *dev, enum scsi_device_state new_state) 2956 { 2957 if (scsi_is_target_device(dev)) 2958 starget_for_each_device(to_scsi_target(dev), &new_state, 2959 device_unblock); 2960 else 2961 device_for_each_child(dev, &new_state, target_unblock); 2962 } 2963 EXPORT_SYMBOL_GPL(scsi_target_unblock); 2964 2965 /** 2966 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt 2967 * @sgl: scatter-gather list 2968 * @sg_count: number of segments in sg 2969 * @offset: offset in bytes into sg, on return offset into the mapped area 2970 * @len: bytes to map, on return number of bytes mapped 2971 * 2972 * Returns virtual address of the start of the mapped page 2973 */ 2974 void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count, 2975 size_t *offset, size_t *len) 2976 { 2977 int i; 2978 size_t sg_len = 0, len_complete = 0; 2979 struct scatterlist *sg; 2980 struct page *page; 2981 2982 WARN_ON(!irqs_disabled()); 2983 2984 for_each_sg(sgl, sg, sg_count, i) { 2985 len_complete = sg_len; /* Complete sg-entries */ 2986 sg_len += sg->length; 2987 if (sg_len > *offset) 2988 break; 2989 } 2990 2991 if (unlikely(i == sg_count)) { 2992 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, " 2993 "elements %d\n", 2994 __func__, sg_len, *offset, sg_count); 2995 WARN_ON(1); 2996 return NULL; 2997 } 2998 2999 /* Offset starting from the beginning of first page in this sg-entry */ 3000 *offset = *offset - len_complete + sg->offset; 3001 3002 /* Assumption: contiguous pages can be accessed as "page + i" */ 3003 page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT)); 3004 *offset &= ~PAGE_MASK; 3005 3006 /* Bytes in this sg-entry from *offset to the end of the page */ 3007 sg_len = PAGE_SIZE - *offset; 3008 if (*len > sg_len) 3009 *len = sg_len; 3010 3011 return kmap_atomic(page); 3012 } 3013 EXPORT_SYMBOL(scsi_kmap_atomic_sg); 3014 3015 /** 3016 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg 3017 * @virt: virtual address to be unmapped 3018 */ 3019 void scsi_kunmap_atomic_sg(void *virt) 3020 { 3021 kunmap_atomic(virt); 3022 } 3023 EXPORT_SYMBOL(scsi_kunmap_atomic_sg); 3024 3025 void sdev_disable_disk_events(struct scsi_device *sdev) 3026 { 3027 atomic_inc(&sdev->disk_events_disable_depth); 3028 } 3029 EXPORT_SYMBOL(sdev_disable_disk_events); 3030 3031 void sdev_enable_disk_events(struct scsi_device *sdev) 3032 { 3033 if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0)) 3034 return; 3035 atomic_dec(&sdev->disk_events_disable_depth); 3036 } 3037 EXPORT_SYMBOL(sdev_enable_disk_events); 3038