1 /* 2 * Linux driver for VMware's para-virtualized SCSI HBA. 3 * 4 * Copyright (C) 2008-2014, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Maintained by: Arvind Kumar <arvindkumar@vmware.com> 21 * 22 */ 23 24 #include <linux/kernel.h> 25 #include <linux/module.h> 26 #include <linux/interrupt.h> 27 #include <linux/slab.h> 28 #include <linux/workqueue.h> 29 #include <linux/pci.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_host.h> 33 #include <scsi/scsi_cmnd.h> 34 #include <scsi/scsi_device.h> 35 #include <scsi/scsi_tcq.h> 36 37 #include "vmw_pvscsi.h" 38 39 #define PVSCSI_LINUX_DRIVER_DESC "VMware PVSCSI driver" 40 41 MODULE_DESCRIPTION(PVSCSI_LINUX_DRIVER_DESC); 42 MODULE_AUTHOR("VMware, Inc."); 43 MODULE_LICENSE("GPL"); 44 MODULE_VERSION(PVSCSI_DRIVER_VERSION_STRING); 45 46 #define PVSCSI_DEFAULT_NUM_PAGES_PER_RING 8 47 #define PVSCSI_DEFAULT_NUM_PAGES_MSG_RING 1 48 #define PVSCSI_DEFAULT_QUEUE_DEPTH 254 49 #define SGL_SIZE PAGE_SIZE 50 51 struct pvscsi_sg_list { 52 struct PVSCSISGElement sge[PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT]; 53 }; 54 55 struct pvscsi_ctx { 56 /* 57 * The index of the context in cmd_map serves as the context ID for a 58 * 1-to-1 mapping completions back to requests. 59 */ 60 struct scsi_cmnd *cmd; 61 struct pvscsi_sg_list *sgl; 62 struct list_head list; 63 dma_addr_t dataPA; 64 dma_addr_t sensePA; 65 dma_addr_t sglPA; 66 struct completion *abort_cmp; 67 }; 68 69 struct pvscsi_adapter { 70 char *mmioBase; 71 unsigned int irq; 72 u8 rev; 73 bool use_msi; 74 bool use_msix; 75 bool use_msg; 76 bool use_req_threshold; 77 78 spinlock_t hw_lock; 79 80 struct workqueue_struct *workqueue; 81 struct work_struct work; 82 83 struct PVSCSIRingReqDesc *req_ring; 84 unsigned req_pages; 85 unsigned req_depth; 86 dma_addr_t reqRingPA; 87 88 struct PVSCSIRingCmpDesc *cmp_ring; 89 unsigned cmp_pages; 90 dma_addr_t cmpRingPA; 91 92 struct PVSCSIRingMsgDesc *msg_ring; 93 unsigned msg_pages; 94 dma_addr_t msgRingPA; 95 96 struct PVSCSIRingsState *rings_state; 97 dma_addr_t ringStatePA; 98 99 struct pci_dev *dev; 100 struct Scsi_Host *host; 101 102 struct list_head cmd_pool; 103 struct pvscsi_ctx *cmd_map; 104 }; 105 106 107 /* Command line parameters */ 108 static int pvscsi_ring_pages; 109 static int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING; 110 static int pvscsi_cmd_per_lun = PVSCSI_DEFAULT_QUEUE_DEPTH; 111 static bool pvscsi_disable_msi; 112 static bool pvscsi_disable_msix; 113 static bool pvscsi_use_msg = true; 114 static bool pvscsi_use_req_threshold = true; 115 116 #define PVSCSI_RW (S_IRUSR | S_IWUSR) 117 118 module_param_named(ring_pages, pvscsi_ring_pages, int, PVSCSI_RW); 119 MODULE_PARM_DESC(ring_pages, "Number of pages per req/cmp ring - (default=" 120 __stringify(PVSCSI_DEFAULT_NUM_PAGES_PER_RING) 121 "[up to 16 targets]," 122 __stringify(PVSCSI_SETUP_RINGS_MAX_NUM_PAGES) 123 "[for 16+ targets])"); 124 125 module_param_named(msg_ring_pages, pvscsi_msg_ring_pages, int, PVSCSI_RW); 126 MODULE_PARM_DESC(msg_ring_pages, "Number of pages for the msg ring - (default=" 127 __stringify(PVSCSI_DEFAULT_NUM_PAGES_MSG_RING) ")"); 128 129 module_param_named(cmd_per_lun, pvscsi_cmd_per_lun, int, PVSCSI_RW); 130 MODULE_PARM_DESC(cmd_per_lun, "Maximum commands per lun - (default=" 131 __stringify(PVSCSI_DEFAULT_QUEUE_DEPTH) ")"); 132 133 module_param_named(disable_msi, pvscsi_disable_msi, bool, PVSCSI_RW); 134 MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)"); 135 136 module_param_named(disable_msix, pvscsi_disable_msix, bool, PVSCSI_RW); 137 MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)"); 138 139 module_param_named(use_msg, pvscsi_use_msg, bool, PVSCSI_RW); 140 MODULE_PARM_DESC(use_msg, "Use msg ring when available - (default=1)"); 141 142 module_param_named(use_req_threshold, pvscsi_use_req_threshold, 143 bool, PVSCSI_RW); 144 MODULE_PARM_DESC(use_req_threshold, "Use driver-based request coalescing if configured - (default=1)"); 145 146 static const struct pci_device_id pvscsi_pci_tbl[] = { 147 { PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_PVSCSI) }, 148 { 0 } 149 }; 150 151 MODULE_DEVICE_TABLE(pci, pvscsi_pci_tbl); 152 153 static struct device * 154 pvscsi_dev(const struct pvscsi_adapter *adapter) 155 { 156 return &(adapter->dev->dev); 157 } 158 159 static struct pvscsi_ctx * 160 pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd) 161 { 162 struct pvscsi_ctx *ctx, *end; 163 164 end = &adapter->cmd_map[adapter->req_depth]; 165 for (ctx = adapter->cmd_map; ctx < end; ctx++) 166 if (ctx->cmd == cmd) 167 return ctx; 168 169 return NULL; 170 } 171 172 static struct pvscsi_ctx * 173 pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd) 174 { 175 struct pvscsi_ctx *ctx; 176 177 if (list_empty(&adapter->cmd_pool)) 178 return NULL; 179 180 ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list); 181 ctx->cmd = cmd; 182 list_del(&ctx->list); 183 184 return ctx; 185 } 186 187 static void pvscsi_release_context(struct pvscsi_adapter *adapter, 188 struct pvscsi_ctx *ctx) 189 { 190 ctx->cmd = NULL; 191 ctx->abort_cmp = NULL; 192 list_add(&ctx->list, &adapter->cmd_pool); 193 } 194 195 /* 196 * Map a pvscsi_ctx struct to a context ID field value; we map to a simple 197 * non-zero integer. ctx always points to an entry in cmd_map array, hence 198 * the return value is always >=1. 199 */ 200 static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter, 201 const struct pvscsi_ctx *ctx) 202 { 203 return ctx - adapter->cmd_map + 1; 204 } 205 206 static struct pvscsi_ctx * 207 pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context) 208 { 209 return &adapter->cmd_map[context - 1]; 210 } 211 212 static void pvscsi_reg_write(const struct pvscsi_adapter *adapter, 213 u32 offset, u32 val) 214 { 215 writel(val, adapter->mmioBase + offset); 216 } 217 218 static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset) 219 { 220 return readl(adapter->mmioBase + offset); 221 } 222 223 static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter) 224 { 225 return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS); 226 } 227 228 static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter, 229 u32 val) 230 { 231 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val); 232 } 233 234 static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter) 235 { 236 u32 intr_bits; 237 238 intr_bits = PVSCSI_INTR_CMPL_MASK; 239 if (adapter->use_msg) 240 intr_bits |= PVSCSI_INTR_MSG_MASK; 241 242 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits); 243 } 244 245 static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter) 246 { 247 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0); 248 } 249 250 static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter, 251 u32 cmd, const void *desc, size_t len) 252 { 253 const u32 *ptr = desc; 254 size_t i; 255 256 len /= sizeof(*ptr); 257 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd); 258 for (i = 0; i < len; i++) 259 pvscsi_reg_write(adapter, 260 PVSCSI_REG_OFFSET_COMMAND_DATA, ptr[i]); 261 } 262 263 static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter, 264 const struct pvscsi_ctx *ctx) 265 { 266 struct PVSCSICmdDescAbortCmd cmd = { 0 }; 267 268 cmd.target = ctx->cmd->device->id; 269 cmd.context = pvscsi_map_context(adapter, ctx); 270 271 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd)); 272 } 273 274 static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter) 275 { 276 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0); 277 } 278 279 static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter) 280 { 281 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0); 282 } 283 284 static int scsi_is_rw(unsigned char op) 285 { 286 return op == READ_6 || op == WRITE_6 || 287 op == READ_10 || op == WRITE_10 || 288 op == READ_12 || op == WRITE_12 || 289 op == READ_16 || op == WRITE_16; 290 } 291 292 static void pvscsi_kick_io(const struct pvscsi_adapter *adapter, 293 unsigned char op) 294 { 295 if (scsi_is_rw(op)) { 296 struct PVSCSIRingsState *s = adapter->rings_state; 297 298 if (!adapter->use_req_threshold || 299 s->reqProdIdx - s->reqConsIdx >= s->reqCallThreshold) 300 pvscsi_kick_rw_io(adapter); 301 } else { 302 pvscsi_process_request_ring(adapter); 303 } 304 } 305 306 static void ll_adapter_reset(const struct pvscsi_adapter *adapter) 307 { 308 dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter); 309 310 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0); 311 } 312 313 static void ll_bus_reset(const struct pvscsi_adapter *adapter) 314 { 315 dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter); 316 317 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0); 318 } 319 320 static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target) 321 { 322 struct PVSCSICmdDescResetDevice cmd = { 0 }; 323 324 dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target); 325 326 cmd.target = target; 327 328 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE, 329 &cmd, sizeof(cmd)); 330 } 331 332 static void pvscsi_create_sg(struct pvscsi_ctx *ctx, 333 struct scatterlist *sg, unsigned count) 334 { 335 unsigned i; 336 struct PVSCSISGElement *sge; 337 338 BUG_ON(count > PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT); 339 340 sge = &ctx->sgl->sge[0]; 341 for (i = 0; i < count; i++, sg++) { 342 sge[i].addr = sg_dma_address(sg); 343 sge[i].length = sg_dma_len(sg); 344 sge[i].flags = 0; 345 } 346 } 347 348 /* 349 * Map all data buffers for a command into PCI space and 350 * setup the scatter/gather list if needed. 351 */ 352 static int pvscsi_map_buffers(struct pvscsi_adapter *adapter, 353 struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd, 354 struct PVSCSIRingReqDesc *e) 355 { 356 unsigned count; 357 unsigned bufflen = scsi_bufflen(cmd); 358 struct scatterlist *sg; 359 360 e->dataLen = bufflen; 361 e->dataAddr = 0; 362 if (bufflen == 0) 363 return 0; 364 365 sg = scsi_sglist(cmd); 366 count = scsi_sg_count(cmd); 367 if (count != 0) { 368 int segs = scsi_dma_map(cmd); 369 370 if (segs == -ENOMEM) { 371 scmd_printk(KERN_ERR, cmd, 372 "vmw_pvscsi: Failed to map cmd sglist for DMA.\n"); 373 return -ENOMEM; 374 } else if (segs > 1) { 375 pvscsi_create_sg(ctx, sg, segs); 376 377 e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST; 378 ctx->sglPA = pci_map_single(adapter->dev, ctx->sgl, 379 SGL_SIZE, PCI_DMA_TODEVICE); 380 if (pci_dma_mapping_error(adapter->dev, ctx->sglPA)) { 381 scmd_printk(KERN_ERR, cmd, 382 "vmw_pvscsi: Failed to map ctx sglist for DMA.\n"); 383 scsi_dma_unmap(cmd); 384 ctx->sglPA = 0; 385 return -ENOMEM; 386 } 387 e->dataAddr = ctx->sglPA; 388 } else 389 e->dataAddr = sg_dma_address(sg); 390 } else { 391 /* 392 * In case there is no S/G list, scsi_sglist points 393 * directly to the buffer. 394 */ 395 ctx->dataPA = pci_map_single(adapter->dev, sg, bufflen, 396 cmd->sc_data_direction); 397 if (pci_dma_mapping_error(adapter->dev, ctx->dataPA)) { 398 scmd_printk(KERN_ERR, cmd, 399 "vmw_pvscsi: Failed to map direct data buffer for DMA.\n"); 400 return -ENOMEM; 401 } 402 e->dataAddr = ctx->dataPA; 403 } 404 405 return 0; 406 } 407 408 static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter, 409 struct pvscsi_ctx *ctx) 410 { 411 struct scsi_cmnd *cmd; 412 unsigned bufflen; 413 414 cmd = ctx->cmd; 415 bufflen = scsi_bufflen(cmd); 416 417 if (bufflen != 0) { 418 unsigned count = scsi_sg_count(cmd); 419 420 if (count != 0) { 421 scsi_dma_unmap(cmd); 422 if (ctx->sglPA) { 423 pci_unmap_single(adapter->dev, ctx->sglPA, 424 SGL_SIZE, PCI_DMA_TODEVICE); 425 ctx->sglPA = 0; 426 } 427 } else 428 pci_unmap_single(adapter->dev, ctx->dataPA, bufflen, 429 cmd->sc_data_direction); 430 } 431 if (cmd->sense_buffer) 432 pci_unmap_single(adapter->dev, ctx->sensePA, 433 SCSI_SENSE_BUFFERSIZE, PCI_DMA_FROMDEVICE); 434 } 435 436 static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter) 437 { 438 adapter->rings_state = pci_alloc_consistent(adapter->dev, PAGE_SIZE, 439 &adapter->ringStatePA); 440 if (!adapter->rings_state) 441 return -ENOMEM; 442 443 adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING, 444 pvscsi_ring_pages); 445 adapter->req_depth = adapter->req_pages 446 * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE; 447 adapter->req_ring = pci_alloc_consistent(adapter->dev, 448 adapter->req_pages * PAGE_SIZE, 449 &adapter->reqRingPA); 450 if (!adapter->req_ring) 451 return -ENOMEM; 452 453 adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING, 454 pvscsi_ring_pages); 455 adapter->cmp_ring = pci_alloc_consistent(adapter->dev, 456 adapter->cmp_pages * PAGE_SIZE, 457 &adapter->cmpRingPA); 458 if (!adapter->cmp_ring) 459 return -ENOMEM; 460 461 BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE)); 462 BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE)); 463 BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE)); 464 465 if (!adapter->use_msg) 466 return 0; 467 468 adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING, 469 pvscsi_msg_ring_pages); 470 adapter->msg_ring = pci_alloc_consistent(adapter->dev, 471 adapter->msg_pages * PAGE_SIZE, 472 &adapter->msgRingPA); 473 if (!adapter->msg_ring) 474 return -ENOMEM; 475 BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE)); 476 477 return 0; 478 } 479 480 static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter) 481 { 482 struct PVSCSICmdDescSetupRings cmd = { 0 }; 483 dma_addr_t base; 484 unsigned i; 485 486 cmd.ringsStatePPN = adapter->ringStatePA >> PAGE_SHIFT; 487 cmd.reqRingNumPages = adapter->req_pages; 488 cmd.cmpRingNumPages = adapter->cmp_pages; 489 490 base = adapter->reqRingPA; 491 for (i = 0; i < adapter->req_pages; i++) { 492 cmd.reqRingPPNs[i] = base >> PAGE_SHIFT; 493 base += PAGE_SIZE; 494 } 495 496 base = adapter->cmpRingPA; 497 for (i = 0; i < adapter->cmp_pages; i++) { 498 cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT; 499 base += PAGE_SIZE; 500 } 501 502 memset(adapter->rings_state, 0, PAGE_SIZE); 503 memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE); 504 memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE); 505 506 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS, 507 &cmd, sizeof(cmd)); 508 509 if (adapter->use_msg) { 510 struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 }; 511 512 cmd_msg.numPages = adapter->msg_pages; 513 514 base = adapter->msgRingPA; 515 for (i = 0; i < adapter->msg_pages; i++) { 516 cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT; 517 base += PAGE_SIZE; 518 } 519 memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE); 520 521 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING, 522 &cmd_msg, sizeof(cmd_msg)); 523 } 524 } 525 526 static int pvscsi_change_queue_depth(struct scsi_device *sdev, int qdepth) 527 { 528 if (!sdev->tagged_supported) 529 qdepth = 1; 530 return scsi_change_queue_depth(sdev, qdepth); 531 } 532 533 /* 534 * Pull a completion descriptor off and pass the completion back 535 * to the SCSI mid layer. 536 */ 537 static void pvscsi_complete_request(struct pvscsi_adapter *adapter, 538 const struct PVSCSIRingCmpDesc *e) 539 { 540 struct pvscsi_ctx *ctx; 541 struct scsi_cmnd *cmd; 542 struct completion *abort_cmp; 543 u32 btstat = e->hostStatus; 544 u32 sdstat = e->scsiStatus; 545 546 ctx = pvscsi_get_context(adapter, e->context); 547 cmd = ctx->cmd; 548 abort_cmp = ctx->abort_cmp; 549 pvscsi_unmap_buffers(adapter, ctx); 550 pvscsi_release_context(adapter, ctx); 551 if (abort_cmp) { 552 /* 553 * The command was requested to be aborted. Just signal that 554 * the request completed and swallow the actual cmd completion 555 * here. The abort handler will post a completion for this 556 * command indicating that it got successfully aborted. 557 */ 558 complete(abort_cmp); 559 return; 560 } 561 562 cmd->result = 0; 563 if (sdstat != SAM_STAT_GOOD && 564 (btstat == BTSTAT_SUCCESS || 565 btstat == BTSTAT_LINKED_COMMAND_COMPLETED || 566 btstat == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG)) { 567 cmd->result = (DID_OK << 16) | sdstat; 568 if (sdstat == SAM_STAT_CHECK_CONDITION && cmd->sense_buffer) 569 cmd->result |= (DRIVER_SENSE << 24); 570 } else 571 switch (btstat) { 572 case BTSTAT_SUCCESS: 573 case BTSTAT_LINKED_COMMAND_COMPLETED: 574 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG: 575 /* If everything went fine, let's move on.. */ 576 cmd->result = (DID_OK << 16); 577 break; 578 579 case BTSTAT_DATARUN: 580 case BTSTAT_DATA_UNDERRUN: 581 /* Report residual data in underruns */ 582 scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen); 583 cmd->result = (DID_ERROR << 16); 584 break; 585 586 case BTSTAT_SELTIMEO: 587 /* Our emulation returns this for non-connected devs */ 588 cmd->result = (DID_BAD_TARGET << 16); 589 break; 590 591 case BTSTAT_LUNMISMATCH: 592 case BTSTAT_TAGREJECT: 593 case BTSTAT_BADMSG: 594 cmd->result = (DRIVER_INVALID << 24); 595 /* fall through */ 596 597 case BTSTAT_HAHARDWARE: 598 case BTSTAT_INVPHASE: 599 case BTSTAT_HATIMEOUT: 600 case BTSTAT_NORESPONSE: 601 case BTSTAT_DISCONNECT: 602 case BTSTAT_HASOFTWARE: 603 case BTSTAT_BUSFREE: 604 case BTSTAT_SENSFAILED: 605 cmd->result |= (DID_ERROR << 16); 606 break; 607 608 case BTSTAT_SENTRST: 609 case BTSTAT_RECVRST: 610 case BTSTAT_BUSRESET: 611 cmd->result = (DID_RESET << 16); 612 break; 613 614 case BTSTAT_ABORTQUEUE: 615 cmd->result = (DID_ABORT << 16); 616 break; 617 618 case BTSTAT_SCSIPARITY: 619 cmd->result = (DID_PARITY << 16); 620 break; 621 622 default: 623 cmd->result = (DID_ERROR << 16); 624 scmd_printk(KERN_DEBUG, cmd, 625 "Unknown completion status: 0x%x\n", 626 btstat); 627 } 628 629 dev_dbg(&cmd->device->sdev_gendev, 630 "cmd=%p %x ctx=%p result=0x%x status=0x%x,%x\n", 631 cmd, cmd->cmnd[0], ctx, cmd->result, btstat, sdstat); 632 633 cmd->scsi_done(cmd); 634 } 635 636 /* 637 * barrier usage : Since the PVSCSI device is emulated, there could be cases 638 * where we may want to serialize some accesses between the driver and the 639 * emulation layer. We use compiler barriers instead of the more expensive 640 * memory barriers because PVSCSI is only supported on X86 which has strong 641 * memory access ordering. 642 */ 643 static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter) 644 { 645 struct PVSCSIRingsState *s = adapter->rings_state; 646 struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring; 647 u32 cmp_entries = s->cmpNumEntriesLog2; 648 649 while (s->cmpConsIdx != s->cmpProdIdx) { 650 struct PVSCSIRingCmpDesc *e = ring + (s->cmpConsIdx & 651 MASK(cmp_entries)); 652 /* 653 * This barrier() ensures that *e is not dereferenced while 654 * the device emulation still writes data into the slot. 655 * Since the device emulation advances s->cmpProdIdx only after 656 * updating the slot we want to check it first. 657 */ 658 barrier(); 659 pvscsi_complete_request(adapter, e); 660 /* 661 * This barrier() ensures that compiler doesn't reorder write 662 * to s->cmpConsIdx before the read of (*e) inside 663 * pvscsi_complete_request. Otherwise, device emulation may 664 * overwrite *e before we had a chance to read it. 665 */ 666 barrier(); 667 s->cmpConsIdx++; 668 } 669 } 670 671 /* 672 * Translate a Linux SCSI request into a request ring entry. 673 */ 674 static int pvscsi_queue_ring(struct pvscsi_adapter *adapter, 675 struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd) 676 { 677 struct PVSCSIRingsState *s; 678 struct PVSCSIRingReqDesc *e; 679 struct scsi_device *sdev; 680 u32 req_entries; 681 682 s = adapter->rings_state; 683 sdev = cmd->device; 684 req_entries = s->reqNumEntriesLog2; 685 686 /* 687 * If this condition holds, we might have room on the request ring, but 688 * we might not have room on the completion ring for the response. 689 * However, we have already ruled out this possibility - we would not 690 * have successfully allocated a context if it were true, since we only 691 * have one context per request entry. Check for it anyway, since it 692 * would be a serious bug. 693 */ 694 if (s->reqProdIdx - s->cmpConsIdx >= 1 << req_entries) { 695 scmd_printk(KERN_ERR, cmd, "vmw_pvscsi: " 696 "ring full: reqProdIdx=%d cmpConsIdx=%d\n", 697 s->reqProdIdx, s->cmpConsIdx); 698 return -1; 699 } 700 701 e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries)); 702 703 e->bus = sdev->channel; 704 e->target = sdev->id; 705 memset(e->lun, 0, sizeof(e->lun)); 706 e->lun[1] = sdev->lun; 707 708 if (cmd->sense_buffer) { 709 ctx->sensePA = pci_map_single(adapter->dev, cmd->sense_buffer, 710 SCSI_SENSE_BUFFERSIZE, 711 PCI_DMA_FROMDEVICE); 712 if (pci_dma_mapping_error(adapter->dev, ctx->sensePA)) { 713 scmd_printk(KERN_ERR, cmd, 714 "vmw_pvscsi: Failed to map sense buffer for DMA.\n"); 715 ctx->sensePA = 0; 716 return -ENOMEM; 717 } 718 e->senseAddr = ctx->sensePA; 719 e->senseLen = SCSI_SENSE_BUFFERSIZE; 720 } else { 721 e->senseLen = 0; 722 e->senseAddr = 0; 723 } 724 e->cdbLen = cmd->cmd_len; 725 e->vcpuHint = smp_processor_id(); 726 memcpy(e->cdb, cmd->cmnd, e->cdbLen); 727 728 e->tag = SIMPLE_QUEUE_TAG; 729 730 if (cmd->sc_data_direction == DMA_FROM_DEVICE) 731 e->flags = PVSCSI_FLAG_CMD_DIR_TOHOST; 732 else if (cmd->sc_data_direction == DMA_TO_DEVICE) 733 e->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE; 734 else if (cmd->sc_data_direction == DMA_NONE) 735 e->flags = PVSCSI_FLAG_CMD_DIR_NONE; 736 else 737 e->flags = 0; 738 739 if (pvscsi_map_buffers(adapter, ctx, cmd, e) != 0) { 740 if (cmd->sense_buffer) { 741 pci_unmap_single(adapter->dev, ctx->sensePA, 742 SCSI_SENSE_BUFFERSIZE, 743 PCI_DMA_FROMDEVICE); 744 ctx->sensePA = 0; 745 } 746 return -ENOMEM; 747 } 748 749 e->context = pvscsi_map_context(adapter, ctx); 750 751 barrier(); 752 753 s->reqProdIdx++; 754 755 return 0; 756 } 757 758 static int pvscsi_queue_lck(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *)) 759 { 760 struct Scsi_Host *host = cmd->device->host; 761 struct pvscsi_adapter *adapter = shost_priv(host); 762 struct pvscsi_ctx *ctx; 763 unsigned long flags; 764 765 spin_lock_irqsave(&adapter->hw_lock, flags); 766 767 ctx = pvscsi_acquire_context(adapter, cmd); 768 if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) { 769 if (ctx) 770 pvscsi_release_context(adapter, ctx); 771 spin_unlock_irqrestore(&adapter->hw_lock, flags); 772 return SCSI_MLQUEUE_HOST_BUSY; 773 } 774 775 cmd->scsi_done = done; 776 777 dev_dbg(&cmd->device->sdev_gendev, 778 "queued cmd %p, ctx %p, op=%x\n", cmd, ctx, cmd->cmnd[0]); 779 780 spin_unlock_irqrestore(&adapter->hw_lock, flags); 781 782 pvscsi_kick_io(adapter, cmd->cmnd[0]); 783 784 return 0; 785 } 786 787 static DEF_SCSI_QCMD(pvscsi_queue) 788 789 static int pvscsi_abort(struct scsi_cmnd *cmd) 790 { 791 struct pvscsi_adapter *adapter = shost_priv(cmd->device->host); 792 struct pvscsi_ctx *ctx; 793 unsigned long flags; 794 int result = SUCCESS; 795 DECLARE_COMPLETION_ONSTACK(abort_cmp); 796 797 scmd_printk(KERN_DEBUG, cmd, "task abort on host %u, %p\n", 798 adapter->host->host_no, cmd); 799 800 spin_lock_irqsave(&adapter->hw_lock, flags); 801 802 /* 803 * Poll the completion ring first - we might be trying to abort 804 * a command that is waiting to be dispatched in the completion ring. 805 */ 806 pvscsi_process_completion_ring(adapter); 807 808 /* 809 * If there is no context for the command, it either already succeeded 810 * or else was never properly issued. Not our problem. 811 */ 812 ctx = pvscsi_find_context(adapter, cmd); 813 if (!ctx) { 814 scmd_printk(KERN_DEBUG, cmd, "Failed to abort cmd %p\n", cmd); 815 goto out; 816 } 817 818 /* 819 * Mark that the command has been requested to be aborted and issue 820 * the abort. 821 */ 822 ctx->abort_cmp = &abort_cmp; 823 824 pvscsi_abort_cmd(adapter, ctx); 825 spin_unlock_irqrestore(&adapter->hw_lock, flags); 826 /* Wait for 2 secs for the completion. */ 827 wait_for_completion_timeout(&abort_cmp, msecs_to_jiffies(2000)); 828 spin_lock_irqsave(&adapter->hw_lock, flags); 829 830 if (!completion_done(&abort_cmp)) { 831 /* 832 * Failed to abort the command, unmark the fact that it 833 * was requested to be aborted. 834 */ 835 ctx->abort_cmp = NULL; 836 result = FAILED; 837 scmd_printk(KERN_DEBUG, cmd, 838 "Failed to get completion for aborted cmd %p\n", 839 cmd); 840 goto out; 841 } 842 843 /* 844 * Successfully aborted the command. 845 */ 846 cmd->result = (DID_ABORT << 16); 847 cmd->scsi_done(cmd); 848 849 out: 850 spin_unlock_irqrestore(&adapter->hw_lock, flags); 851 return result; 852 } 853 854 /* 855 * Abort all outstanding requests. This is only safe to use if the completion 856 * ring will never be walked again or the device has been reset, because it 857 * destroys the 1-1 mapping between context field passed to emulation and our 858 * request structure. 859 */ 860 static void pvscsi_reset_all(struct pvscsi_adapter *adapter) 861 { 862 unsigned i; 863 864 for (i = 0; i < adapter->req_depth; i++) { 865 struct pvscsi_ctx *ctx = &adapter->cmd_map[i]; 866 struct scsi_cmnd *cmd = ctx->cmd; 867 if (cmd) { 868 scmd_printk(KERN_ERR, cmd, 869 "Forced reset on cmd %p\n", cmd); 870 pvscsi_unmap_buffers(adapter, ctx); 871 pvscsi_release_context(adapter, ctx); 872 cmd->result = (DID_RESET << 16); 873 cmd->scsi_done(cmd); 874 } 875 } 876 } 877 878 static int pvscsi_host_reset(struct scsi_cmnd *cmd) 879 { 880 struct Scsi_Host *host = cmd->device->host; 881 struct pvscsi_adapter *adapter = shost_priv(host); 882 unsigned long flags; 883 bool use_msg; 884 885 scmd_printk(KERN_INFO, cmd, "SCSI Host reset\n"); 886 887 spin_lock_irqsave(&adapter->hw_lock, flags); 888 889 use_msg = adapter->use_msg; 890 891 if (use_msg) { 892 adapter->use_msg = 0; 893 spin_unlock_irqrestore(&adapter->hw_lock, flags); 894 895 /* 896 * Now that we know that the ISR won't add more work on the 897 * workqueue we can safely flush any outstanding work. 898 */ 899 flush_workqueue(adapter->workqueue); 900 spin_lock_irqsave(&adapter->hw_lock, flags); 901 } 902 903 /* 904 * We're going to tear down the entire ring structure and set it back 905 * up, so stalling new requests until all completions are flushed and 906 * the rings are back in place. 907 */ 908 909 pvscsi_process_request_ring(adapter); 910 911 ll_adapter_reset(adapter); 912 913 /* 914 * Now process any completions. Note we do this AFTER adapter reset, 915 * which is strange, but stops races where completions get posted 916 * between processing the ring and issuing the reset. The backend will 917 * not touch the ring memory after reset, so the immediately pre-reset 918 * completion ring state is still valid. 919 */ 920 pvscsi_process_completion_ring(adapter); 921 922 pvscsi_reset_all(adapter); 923 adapter->use_msg = use_msg; 924 pvscsi_setup_all_rings(adapter); 925 pvscsi_unmask_intr(adapter); 926 927 spin_unlock_irqrestore(&adapter->hw_lock, flags); 928 929 return SUCCESS; 930 } 931 932 static int pvscsi_bus_reset(struct scsi_cmnd *cmd) 933 { 934 struct Scsi_Host *host = cmd->device->host; 935 struct pvscsi_adapter *adapter = shost_priv(host); 936 unsigned long flags; 937 938 scmd_printk(KERN_INFO, cmd, "SCSI Bus reset\n"); 939 940 /* 941 * We don't want to queue new requests for this bus after 942 * flushing all pending requests to emulation, since new 943 * requests could then sneak in during this bus reset phase, 944 * so take the lock now. 945 */ 946 spin_lock_irqsave(&adapter->hw_lock, flags); 947 948 pvscsi_process_request_ring(adapter); 949 ll_bus_reset(adapter); 950 pvscsi_process_completion_ring(adapter); 951 952 spin_unlock_irqrestore(&adapter->hw_lock, flags); 953 954 return SUCCESS; 955 } 956 957 static int pvscsi_device_reset(struct scsi_cmnd *cmd) 958 { 959 struct Scsi_Host *host = cmd->device->host; 960 struct pvscsi_adapter *adapter = shost_priv(host); 961 unsigned long flags; 962 963 scmd_printk(KERN_INFO, cmd, "SCSI device reset on scsi%u:%u\n", 964 host->host_no, cmd->device->id); 965 966 /* 967 * We don't want to queue new requests for this device after flushing 968 * all pending requests to emulation, since new requests could then 969 * sneak in during this device reset phase, so take the lock now. 970 */ 971 spin_lock_irqsave(&adapter->hw_lock, flags); 972 973 pvscsi_process_request_ring(adapter); 974 ll_device_reset(adapter, cmd->device->id); 975 pvscsi_process_completion_ring(adapter); 976 977 spin_unlock_irqrestore(&adapter->hw_lock, flags); 978 979 return SUCCESS; 980 } 981 982 static struct scsi_host_template pvscsi_template; 983 984 static const char *pvscsi_info(struct Scsi_Host *host) 985 { 986 struct pvscsi_adapter *adapter = shost_priv(host); 987 static char buf[256]; 988 989 sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: " 990 "%u/%u/%u pages, cmd_per_lun=%u", adapter->rev, 991 adapter->req_pages, adapter->cmp_pages, adapter->msg_pages, 992 pvscsi_template.cmd_per_lun); 993 994 return buf; 995 } 996 997 static struct scsi_host_template pvscsi_template = { 998 .module = THIS_MODULE, 999 .name = "VMware PVSCSI Host Adapter", 1000 .proc_name = "vmw_pvscsi", 1001 .info = pvscsi_info, 1002 .queuecommand = pvscsi_queue, 1003 .this_id = -1, 1004 .sg_tablesize = PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT, 1005 .dma_boundary = UINT_MAX, 1006 .max_sectors = 0xffff, 1007 .use_clustering = ENABLE_CLUSTERING, 1008 .change_queue_depth = pvscsi_change_queue_depth, 1009 .eh_abort_handler = pvscsi_abort, 1010 .eh_device_reset_handler = pvscsi_device_reset, 1011 .eh_bus_reset_handler = pvscsi_bus_reset, 1012 .eh_host_reset_handler = pvscsi_host_reset, 1013 }; 1014 1015 static void pvscsi_process_msg(const struct pvscsi_adapter *adapter, 1016 const struct PVSCSIRingMsgDesc *e) 1017 { 1018 struct PVSCSIRingsState *s = adapter->rings_state; 1019 struct Scsi_Host *host = adapter->host; 1020 struct scsi_device *sdev; 1021 1022 printk(KERN_INFO "vmw_pvscsi: msg type: 0x%x - MSG RING: %u/%u (%u) \n", 1023 e->type, s->msgProdIdx, s->msgConsIdx, s->msgNumEntriesLog2); 1024 1025 BUILD_BUG_ON(PVSCSI_MSG_LAST != 2); 1026 1027 if (e->type == PVSCSI_MSG_DEV_ADDED) { 1028 struct PVSCSIMsgDescDevStatusChanged *desc; 1029 desc = (struct PVSCSIMsgDescDevStatusChanged *)e; 1030 1031 printk(KERN_INFO 1032 "vmw_pvscsi: msg: device added at scsi%u:%u:%u\n", 1033 desc->bus, desc->target, desc->lun[1]); 1034 1035 if (!scsi_host_get(host)) 1036 return; 1037 1038 sdev = scsi_device_lookup(host, desc->bus, desc->target, 1039 desc->lun[1]); 1040 if (sdev) { 1041 printk(KERN_INFO "vmw_pvscsi: device already exists\n"); 1042 scsi_device_put(sdev); 1043 } else 1044 scsi_add_device(adapter->host, desc->bus, 1045 desc->target, desc->lun[1]); 1046 1047 scsi_host_put(host); 1048 } else if (e->type == PVSCSI_MSG_DEV_REMOVED) { 1049 struct PVSCSIMsgDescDevStatusChanged *desc; 1050 desc = (struct PVSCSIMsgDescDevStatusChanged *)e; 1051 1052 printk(KERN_INFO 1053 "vmw_pvscsi: msg: device removed at scsi%u:%u:%u\n", 1054 desc->bus, desc->target, desc->lun[1]); 1055 1056 if (!scsi_host_get(host)) 1057 return; 1058 1059 sdev = scsi_device_lookup(host, desc->bus, desc->target, 1060 desc->lun[1]); 1061 if (sdev) { 1062 scsi_remove_device(sdev); 1063 scsi_device_put(sdev); 1064 } else 1065 printk(KERN_INFO 1066 "vmw_pvscsi: failed to lookup scsi%u:%u:%u\n", 1067 desc->bus, desc->target, desc->lun[1]); 1068 1069 scsi_host_put(host); 1070 } 1071 } 1072 1073 static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter) 1074 { 1075 struct PVSCSIRingsState *s = adapter->rings_state; 1076 1077 return s->msgProdIdx != s->msgConsIdx; 1078 } 1079 1080 static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter) 1081 { 1082 struct PVSCSIRingsState *s = adapter->rings_state; 1083 struct PVSCSIRingMsgDesc *ring = adapter->msg_ring; 1084 u32 msg_entries = s->msgNumEntriesLog2; 1085 1086 while (pvscsi_msg_pending(adapter)) { 1087 struct PVSCSIRingMsgDesc *e = ring + (s->msgConsIdx & 1088 MASK(msg_entries)); 1089 1090 barrier(); 1091 pvscsi_process_msg(adapter, e); 1092 barrier(); 1093 s->msgConsIdx++; 1094 } 1095 } 1096 1097 static void pvscsi_msg_workqueue_handler(struct work_struct *data) 1098 { 1099 struct pvscsi_adapter *adapter; 1100 1101 adapter = container_of(data, struct pvscsi_adapter, work); 1102 1103 pvscsi_process_msg_ring(adapter); 1104 } 1105 1106 static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter) 1107 { 1108 char name[32]; 1109 1110 if (!pvscsi_use_msg) 1111 return 0; 1112 1113 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, 1114 PVSCSI_CMD_SETUP_MSG_RING); 1115 1116 if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1) 1117 return 0; 1118 1119 snprintf(name, sizeof(name), 1120 "vmw_pvscsi_wq_%u", adapter->host->host_no); 1121 1122 adapter->workqueue = create_singlethread_workqueue(name); 1123 if (!adapter->workqueue) { 1124 printk(KERN_ERR "vmw_pvscsi: failed to create work queue\n"); 1125 return 0; 1126 } 1127 INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler); 1128 1129 return 1; 1130 } 1131 1132 static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter, 1133 bool enable) 1134 { 1135 u32 val; 1136 1137 if (!pvscsi_use_req_threshold) 1138 return false; 1139 1140 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, 1141 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD); 1142 val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS); 1143 if (val == -1) { 1144 printk(KERN_INFO "vmw_pvscsi: device does not support req_threshold\n"); 1145 return false; 1146 } else { 1147 struct PVSCSICmdDescSetupReqCall cmd_msg = { 0 }; 1148 cmd_msg.enable = enable; 1149 printk(KERN_INFO 1150 "vmw_pvscsi: %sabling reqCallThreshold\n", 1151 enable ? "en" : "dis"); 1152 pvscsi_write_cmd_desc(adapter, 1153 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD, 1154 &cmd_msg, sizeof(cmd_msg)); 1155 return pvscsi_reg_read(adapter, 1156 PVSCSI_REG_OFFSET_COMMAND_STATUS) != 0; 1157 } 1158 } 1159 1160 static irqreturn_t pvscsi_isr(int irq, void *devp) 1161 { 1162 struct pvscsi_adapter *adapter = devp; 1163 int handled; 1164 1165 if (adapter->use_msi || adapter->use_msix) 1166 handled = true; 1167 else { 1168 u32 val = pvscsi_read_intr_status(adapter); 1169 handled = (val & PVSCSI_INTR_ALL_SUPPORTED) != 0; 1170 if (handled) 1171 pvscsi_write_intr_status(devp, val); 1172 } 1173 1174 if (handled) { 1175 unsigned long flags; 1176 1177 spin_lock_irqsave(&adapter->hw_lock, flags); 1178 1179 pvscsi_process_completion_ring(adapter); 1180 if (adapter->use_msg && pvscsi_msg_pending(adapter)) 1181 queue_work(adapter->workqueue, &adapter->work); 1182 1183 spin_unlock_irqrestore(&adapter->hw_lock, flags); 1184 } 1185 1186 return IRQ_RETVAL(handled); 1187 } 1188 1189 static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter) 1190 { 1191 struct pvscsi_ctx *ctx = adapter->cmd_map; 1192 unsigned i; 1193 1194 for (i = 0; i < adapter->req_depth; ++i, ++ctx) 1195 free_pages((unsigned long)ctx->sgl, get_order(SGL_SIZE)); 1196 } 1197 1198 static int pvscsi_setup_msix(const struct pvscsi_adapter *adapter, 1199 unsigned int *irq) 1200 { 1201 struct msix_entry entry = { 0, PVSCSI_VECTOR_COMPLETION }; 1202 int ret; 1203 1204 ret = pci_enable_msix_exact(adapter->dev, &entry, 1); 1205 if (ret) 1206 return ret; 1207 1208 *irq = entry.vector; 1209 1210 return 0; 1211 } 1212 1213 static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter) 1214 { 1215 if (adapter->irq) { 1216 free_irq(adapter->irq, adapter); 1217 adapter->irq = 0; 1218 } 1219 if (adapter->use_msi) { 1220 pci_disable_msi(adapter->dev); 1221 adapter->use_msi = 0; 1222 } else if (adapter->use_msix) { 1223 pci_disable_msix(adapter->dev); 1224 adapter->use_msix = 0; 1225 } 1226 } 1227 1228 static void pvscsi_release_resources(struct pvscsi_adapter *adapter) 1229 { 1230 pvscsi_shutdown_intr(adapter); 1231 1232 if (adapter->workqueue) 1233 destroy_workqueue(adapter->workqueue); 1234 1235 if (adapter->mmioBase) 1236 pci_iounmap(adapter->dev, adapter->mmioBase); 1237 1238 pci_release_regions(adapter->dev); 1239 1240 if (adapter->cmd_map) { 1241 pvscsi_free_sgls(adapter); 1242 kfree(adapter->cmd_map); 1243 } 1244 1245 if (adapter->rings_state) 1246 pci_free_consistent(adapter->dev, PAGE_SIZE, 1247 adapter->rings_state, adapter->ringStatePA); 1248 1249 if (adapter->req_ring) 1250 pci_free_consistent(adapter->dev, 1251 adapter->req_pages * PAGE_SIZE, 1252 adapter->req_ring, adapter->reqRingPA); 1253 1254 if (adapter->cmp_ring) 1255 pci_free_consistent(adapter->dev, 1256 adapter->cmp_pages * PAGE_SIZE, 1257 adapter->cmp_ring, adapter->cmpRingPA); 1258 1259 if (adapter->msg_ring) 1260 pci_free_consistent(adapter->dev, 1261 adapter->msg_pages * PAGE_SIZE, 1262 adapter->msg_ring, adapter->msgRingPA); 1263 } 1264 1265 /* 1266 * Allocate scatter gather lists. 1267 * 1268 * These are statically allocated. Trying to be clever was not worth it. 1269 * 1270 * Dynamic allocation can fail, and we can't go deep into the memory 1271 * allocator, since we're a SCSI driver, and trying too hard to allocate 1272 * memory might generate disk I/O. We also don't want to fail disk I/O 1273 * in that case because we can't get an allocation - the I/O could be 1274 * trying to swap out data to free memory. Since that is pathological, 1275 * just use a statically allocated scatter list. 1276 * 1277 */ 1278 static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter) 1279 { 1280 struct pvscsi_ctx *ctx; 1281 int i; 1282 1283 ctx = adapter->cmd_map; 1284 BUILD_BUG_ON(sizeof(struct pvscsi_sg_list) > SGL_SIZE); 1285 1286 for (i = 0; i < adapter->req_depth; ++i, ++ctx) { 1287 ctx->sgl = (void *)__get_free_pages(GFP_KERNEL, 1288 get_order(SGL_SIZE)); 1289 ctx->sglPA = 0; 1290 BUG_ON(!IS_ALIGNED(((unsigned long)ctx->sgl), PAGE_SIZE)); 1291 if (!ctx->sgl) { 1292 for (; i >= 0; --i, --ctx) { 1293 free_pages((unsigned long)ctx->sgl, 1294 get_order(SGL_SIZE)); 1295 ctx->sgl = NULL; 1296 } 1297 return -ENOMEM; 1298 } 1299 } 1300 1301 return 0; 1302 } 1303 1304 /* 1305 * Query the device, fetch the config info and return the 1306 * maximum number of targets on the adapter. In case of 1307 * failure due to any reason return default i.e. 16. 1308 */ 1309 static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter) 1310 { 1311 struct PVSCSICmdDescConfigCmd cmd; 1312 struct PVSCSIConfigPageHeader *header; 1313 struct device *dev; 1314 dma_addr_t configPagePA; 1315 void *config_page; 1316 u32 numPhys = 16; 1317 1318 dev = pvscsi_dev(adapter); 1319 config_page = pci_alloc_consistent(adapter->dev, PAGE_SIZE, 1320 &configPagePA); 1321 if (!config_page) { 1322 dev_warn(dev, "vmw_pvscsi: failed to allocate memory for config page\n"); 1323 goto exit; 1324 } 1325 BUG_ON(configPagePA & ~PAGE_MASK); 1326 1327 /* Fetch config info from the device. */ 1328 cmd.configPageAddress = ((u64)PVSCSI_CONFIG_CONTROLLER_ADDRESS) << 32; 1329 cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER; 1330 cmd.cmpAddr = configPagePA; 1331 cmd._pad = 0; 1332 1333 /* 1334 * Mark the completion page header with error values. If the device 1335 * completes the command successfully, it sets the status values to 1336 * indicate success. 1337 */ 1338 header = config_page; 1339 memset(header, 0, sizeof *header); 1340 header->hostStatus = BTSTAT_INVPARAM; 1341 header->scsiStatus = SDSTAT_CHECK; 1342 1343 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd); 1344 1345 if (header->hostStatus == BTSTAT_SUCCESS && 1346 header->scsiStatus == SDSTAT_GOOD) { 1347 struct PVSCSIConfigPageController *config; 1348 1349 config = config_page; 1350 numPhys = config->numPhys; 1351 } else 1352 dev_warn(dev, "vmw_pvscsi: PVSCSI_CMD_CONFIG failed. hostStatus = 0x%x, scsiStatus = 0x%x\n", 1353 header->hostStatus, header->scsiStatus); 1354 pci_free_consistent(adapter->dev, PAGE_SIZE, config_page, configPagePA); 1355 exit: 1356 return numPhys; 1357 } 1358 1359 static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1360 { 1361 struct pvscsi_adapter *adapter; 1362 struct pvscsi_adapter adapter_temp; 1363 struct Scsi_Host *host = NULL; 1364 unsigned int i; 1365 unsigned long flags = 0; 1366 int error; 1367 u32 max_id; 1368 1369 error = -ENODEV; 1370 1371 if (pci_enable_device(pdev)) 1372 return error; 1373 1374 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0 && 1375 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) { 1376 printk(KERN_INFO "vmw_pvscsi: using 64bit dma\n"); 1377 } else if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) == 0 && 1378 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) == 0) { 1379 printk(KERN_INFO "vmw_pvscsi: using 32bit dma\n"); 1380 } else { 1381 printk(KERN_ERR "vmw_pvscsi: failed to set DMA mask\n"); 1382 goto out_disable_device; 1383 } 1384 1385 /* 1386 * Let's use a temp pvscsi_adapter struct until we find the number of 1387 * targets on the adapter, after that we will switch to the real 1388 * allocated struct. 1389 */ 1390 adapter = &adapter_temp; 1391 memset(adapter, 0, sizeof(*adapter)); 1392 adapter->dev = pdev; 1393 adapter->rev = pdev->revision; 1394 1395 if (pci_request_regions(pdev, "vmw_pvscsi")) { 1396 printk(KERN_ERR "vmw_pvscsi: pci memory selection failed\n"); 1397 goto out_disable_device; 1398 } 1399 1400 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { 1401 if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE_IO)) 1402 continue; 1403 1404 if (pci_resource_len(pdev, i) < PVSCSI_MEM_SPACE_SIZE) 1405 continue; 1406 1407 break; 1408 } 1409 1410 if (i == DEVICE_COUNT_RESOURCE) { 1411 printk(KERN_ERR 1412 "vmw_pvscsi: adapter has no suitable MMIO region\n"); 1413 goto out_release_resources_and_disable; 1414 } 1415 1416 adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE); 1417 1418 if (!adapter->mmioBase) { 1419 printk(KERN_ERR 1420 "vmw_pvscsi: can't iomap for BAR %d memsize %lu\n", 1421 i, PVSCSI_MEM_SPACE_SIZE); 1422 goto out_release_resources_and_disable; 1423 } 1424 1425 pci_set_master(pdev); 1426 1427 /* 1428 * Ask the device for max number of targets before deciding the 1429 * default pvscsi_ring_pages value. 1430 */ 1431 max_id = pvscsi_get_max_targets(adapter); 1432 printk(KERN_INFO "vmw_pvscsi: max_id: %u\n", max_id); 1433 1434 if (pvscsi_ring_pages == 0) 1435 /* 1436 * Set the right default value. Up to 16 it is 8, above it is 1437 * max. 1438 */ 1439 pvscsi_ring_pages = (max_id > 16) ? 1440 PVSCSI_SETUP_RINGS_MAX_NUM_PAGES : 1441 PVSCSI_DEFAULT_NUM_PAGES_PER_RING; 1442 printk(KERN_INFO 1443 "vmw_pvscsi: setting ring_pages to %d\n", 1444 pvscsi_ring_pages); 1445 1446 pvscsi_template.can_queue = 1447 min(PVSCSI_MAX_NUM_PAGES_REQ_RING, pvscsi_ring_pages) * 1448 PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE; 1449 pvscsi_template.cmd_per_lun = 1450 min(pvscsi_template.can_queue, pvscsi_cmd_per_lun); 1451 host = scsi_host_alloc(&pvscsi_template, sizeof(struct pvscsi_adapter)); 1452 if (!host) { 1453 printk(KERN_ERR "vmw_pvscsi: failed to allocate host\n"); 1454 goto out_release_resources_and_disable; 1455 } 1456 1457 /* 1458 * Let's use the real pvscsi_adapter struct here onwards. 1459 */ 1460 adapter = shost_priv(host); 1461 memset(adapter, 0, sizeof(*adapter)); 1462 adapter->dev = pdev; 1463 adapter->host = host; 1464 /* 1465 * Copy back what we already have to the allocated adapter struct. 1466 */ 1467 adapter->rev = adapter_temp.rev; 1468 adapter->mmioBase = adapter_temp.mmioBase; 1469 1470 spin_lock_init(&adapter->hw_lock); 1471 host->max_channel = 0; 1472 host->max_lun = 1; 1473 host->max_cmd_len = 16; 1474 host->max_id = max_id; 1475 1476 pci_set_drvdata(pdev, host); 1477 1478 ll_adapter_reset(adapter); 1479 1480 adapter->use_msg = pvscsi_setup_msg_workqueue(adapter); 1481 1482 error = pvscsi_allocate_rings(adapter); 1483 if (error) { 1484 printk(KERN_ERR "vmw_pvscsi: unable to allocate ring memory\n"); 1485 goto out_release_resources; 1486 } 1487 1488 /* 1489 * From this point on we should reset the adapter if anything goes 1490 * wrong. 1491 */ 1492 pvscsi_setup_all_rings(adapter); 1493 1494 adapter->cmd_map = kcalloc(adapter->req_depth, 1495 sizeof(struct pvscsi_ctx), GFP_KERNEL); 1496 if (!adapter->cmd_map) { 1497 printk(KERN_ERR "vmw_pvscsi: failed to allocate memory.\n"); 1498 error = -ENOMEM; 1499 goto out_reset_adapter; 1500 } 1501 1502 INIT_LIST_HEAD(&adapter->cmd_pool); 1503 for (i = 0; i < adapter->req_depth; i++) { 1504 struct pvscsi_ctx *ctx = adapter->cmd_map + i; 1505 list_add(&ctx->list, &adapter->cmd_pool); 1506 } 1507 1508 error = pvscsi_allocate_sg(adapter); 1509 if (error) { 1510 printk(KERN_ERR "vmw_pvscsi: unable to allocate s/g table\n"); 1511 goto out_reset_adapter; 1512 } 1513 1514 if (!pvscsi_disable_msix && 1515 pvscsi_setup_msix(adapter, &adapter->irq) == 0) { 1516 printk(KERN_INFO "vmw_pvscsi: using MSI-X\n"); 1517 adapter->use_msix = 1; 1518 } else if (!pvscsi_disable_msi && pci_enable_msi(pdev) == 0) { 1519 printk(KERN_INFO "vmw_pvscsi: using MSI\n"); 1520 adapter->use_msi = 1; 1521 adapter->irq = pdev->irq; 1522 } else { 1523 printk(KERN_INFO "vmw_pvscsi: using INTx\n"); 1524 adapter->irq = pdev->irq; 1525 flags = IRQF_SHARED; 1526 } 1527 1528 adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true); 1529 printk(KERN_DEBUG "vmw_pvscsi: driver-based request coalescing %sabled\n", 1530 adapter->use_req_threshold ? "en" : "dis"); 1531 1532 error = request_irq(adapter->irq, pvscsi_isr, flags, 1533 "vmw_pvscsi", adapter); 1534 if (error) { 1535 printk(KERN_ERR 1536 "vmw_pvscsi: unable to request IRQ: %d\n", error); 1537 adapter->irq = 0; 1538 goto out_reset_adapter; 1539 } 1540 1541 error = scsi_add_host(host, &pdev->dev); 1542 if (error) { 1543 printk(KERN_ERR 1544 "vmw_pvscsi: scsi_add_host failed: %d\n", error); 1545 goto out_reset_adapter; 1546 } 1547 1548 dev_info(&pdev->dev, "VMware PVSCSI rev %d host #%u\n", 1549 adapter->rev, host->host_no); 1550 1551 pvscsi_unmask_intr(adapter); 1552 1553 scsi_scan_host(host); 1554 1555 return 0; 1556 1557 out_reset_adapter: 1558 ll_adapter_reset(adapter); 1559 out_release_resources: 1560 pvscsi_release_resources(adapter); 1561 scsi_host_put(host); 1562 out_disable_device: 1563 pci_disable_device(pdev); 1564 1565 return error; 1566 1567 out_release_resources_and_disable: 1568 pvscsi_release_resources(adapter); 1569 goto out_disable_device; 1570 } 1571 1572 static void __pvscsi_shutdown(struct pvscsi_adapter *adapter) 1573 { 1574 pvscsi_mask_intr(adapter); 1575 1576 if (adapter->workqueue) 1577 flush_workqueue(adapter->workqueue); 1578 1579 pvscsi_shutdown_intr(adapter); 1580 1581 pvscsi_process_request_ring(adapter); 1582 pvscsi_process_completion_ring(adapter); 1583 ll_adapter_reset(adapter); 1584 } 1585 1586 static void pvscsi_shutdown(struct pci_dev *dev) 1587 { 1588 struct Scsi_Host *host = pci_get_drvdata(dev); 1589 struct pvscsi_adapter *adapter = shost_priv(host); 1590 1591 __pvscsi_shutdown(adapter); 1592 } 1593 1594 static void pvscsi_remove(struct pci_dev *pdev) 1595 { 1596 struct Scsi_Host *host = pci_get_drvdata(pdev); 1597 struct pvscsi_adapter *adapter = shost_priv(host); 1598 1599 scsi_remove_host(host); 1600 1601 __pvscsi_shutdown(adapter); 1602 pvscsi_release_resources(adapter); 1603 1604 scsi_host_put(host); 1605 1606 pci_disable_device(pdev); 1607 } 1608 1609 static struct pci_driver pvscsi_pci_driver = { 1610 .name = "vmw_pvscsi", 1611 .id_table = pvscsi_pci_tbl, 1612 .probe = pvscsi_probe, 1613 .remove = pvscsi_remove, 1614 .shutdown = pvscsi_shutdown, 1615 }; 1616 1617 static int __init pvscsi_init(void) 1618 { 1619 pr_info("%s - version %s\n", 1620 PVSCSI_LINUX_DRIVER_DESC, PVSCSI_DRIVER_VERSION_STRING); 1621 return pci_register_driver(&pvscsi_pci_driver); 1622 } 1623 1624 static void __exit pvscsi_exit(void) 1625 { 1626 pci_unregister_driver(&pvscsi_pci_driver); 1627 } 1628 1629 module_init(pvscsi_init); 1630 module_exit(pvscsi_exit); 1631