1 /* 2 * This is the Fusion MPT base driver providing common API layer interface 3 * for access to MPT (Message Passing Technology) firmware. 4 * 5 * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c 6 * Copyright (C) 2012-2014 LSI Corporation 7 * Copyright (C) 2013-2014 Avago Technologies 8 * (mailto: MPT-FusionLinux.pdl@avagotech.com) 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 2 13 * of the License, or (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * NO WARRANTY 21 * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR 22 * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT 23 * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, 24 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is 25 * solely responsible for determining the appropriateness of using and 26 * distributing the Program and assumes all risks associated with its 27 * exercise of rights under this Agreement, including but not limited to 28 * the risks and costs of program errors, damage to or loss of data, 29 * programs or equipment, and unavailability or interruption of operations. 30 31 * DISCLAIMER OF LIABILITY 32 * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY 33 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND 35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 37 * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED 38 * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES 39 40 * You should have received a copy of the GNU General Public License 41 * along with this program; if not, write to the Free Software 42 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, 43 * USA. 44 */ 45 46 #include <linux/kernel.h> 47 #include <linux/module.h> 48 #include <linux/errno.h> 49 #include <linux/init.h> 50 #include <linux/slab.h> 51 #include <linux/types.h> 52 #include <linux/pci.h> 53 #include <linux/kdev_t.h> 54 #include <linux/blkdev.h> 55 #include <linux/delay.h> 56 #include <linux/interrupt.h> 57 #include <linux/dma-mapping.h> 58 #include <linux/io.h> 59 #include <linux/time.h> 60 #include <linux/ktime.h> 61 #include <linux/kthread.h> 62 #include <asm/page.h> /* To get host page size per arch */ 63 #include <linux/aer.h> 64 65 66 #include "mpt3sas_base.h" 67 68 static MPT_CALLBACK mpt_callbacks[MPT_MAX_CALLBACKS]; 69 70 71 #define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */ 72 73 /* maximum controller queue depth */ 74 #define MAX_HBA_QUEUE_DEPTH 30000 75 #define MAX_CHAIN_DEPTH 100000 76 static int max_queue_depth = -1; 77 module_param(max_queue_depth, int, 0444); 78 MODULE_PARM_DESC(max_queue_depth, " max controller queue depth "); 79 80 static int max_sgl_entries = -1; 81 module_param(max_sgl_entries, int, 0444); 82 MODULE_PARM_DESC(max_sgl_entries, " max sg entries "); 83 84 static int msix_disable = -1; 85 module_param(msix_disable, int, 0444); 86 MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)"); 87 88 static int smp_affinity_enable = 1; 89 module_param(smp_affinity_enable, int, 0444); 90 MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)"); 91 92 static int max_msix_vectors = -1; 93 module_param(max_msix_vectors, int, 0444); 94 MODULE_PARM_DESC(max_msix_vectors, 95 " max msix vectors"); 96 97 static int irqpoll_weight = -1; 98 module_param(irqpoll_weight, int, 0444); 99 MODULE_PARM_DESC(irqpoll_weight, 100 "irq poll weight (default= one fourth of HBA queue depth)"); 101 102 static int mpt3sas_fwfault_debug; 103 MODULE_PARM_DESC(mpt3sas_fwfault_debug, 104 " enable detection of firmware fault and halt firmware - (default=0)"); 105 106 static int perf_mode = -1; 107 module_param(perf_mode, int, 0444); 108 MODULE_PARM_DESC(perf_mode, 109 "Performance mode (only for Aero/Sea Generation), options:\n\t\t" 110 "0 - balanced: high iops mode is enabled &\n\t\t" 111 "interrupt coalescing is enabled only on high iops queues,\n\t\t" 112 "1 - iops: high iops mode is disabled &\n\t\t" 113 "interrupt coalescing is enabled on all queues,\n\t\t" 114 "2 - latency: high iops mode is disabled &\n\t\t" 115 "interrupt coalescing is enabled on all queues with timeout value 0xA,\n" 116 "\t\tdefault - default perf_mode is 'balanced'" 117 ); 118 119 static int poll_queues; 120 module_param(poll_queues, int, 0444); 121 MODULE_PARM_DESC(poll_queues, "Number of queues to be use for io_uring poll mode.\n\t\t" 122 "This parameter is effective only if host_tagset_enable=1. &\n\t\t" 123 "when poll_queues are enabled then &\n\t\t" 124 "perf_mode is set to latency mode. &\n\t\t" 125 ); 126 127 enum mpt3sas_perf_mode { 128 MPT_PERF_MODE_DEFAULT = -1, 129 MPT_PERF_MODE_BALANCED = 0, 130 MPT_PERF_MODE_IOPS = 1, 131 MPT_PERF_MODE_LATENCY = 2, 132 }; 133 134 static int 135 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, 136 u32 ioc_state, int timeout); 137 static int 138 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc); 139 static void 140 _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc); 141 142 /** 143 * mpt3sas_base_check_cmd_timeout - Function 144 * to check timeout and command termination due 145 * to Host reset. 146 * 147 * @ioc: per adapter object. 148 * @status: Status of issued command. 149 * @mpi_request:mf request pointer. 150 * @sz: size of buffer. 151 * 152 * Return: 1/0 Reset to be done or Not 153 */ 154 u8 155 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc, 156 u8 status, void *mpi_request, int sz) 157 { 158 u8 issue_reset = 0; 159 160 if (!(status & MPT3_CMD_RESET)) 161 issue_reset = 1; 162 163 ioc_err(ioc, "Command %s\n", 164 issue_reset == 0 ? "terminated due to Host Reset" : "Timeout"); 165 _debug_dump_mf(mpi_request, sz); 166 167 return issue_reset; 168 } 169 170 /** 171 * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug. 172 * @val: ? 173 * @kp: ? 174 * 175 * Return: ? 176 */ 177 static int 178 _scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp) 179 { 180 int ret = param_set_int(val, kp); 181 struct MPT3SAS_ADAPTER *ioc; 182 183 if (ret) 184 return ret; 185 186 /* global ioc spinlock to protect controller list on list operations */ 187 pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug); 188 spin_lock(&gioc_lock); 189 list_for_each_entry(ioc, &mpt3sas_ioc_list, list) 190 ioc->fwfault_debug = mpt3sas_fwfault_debug; 191 spin_unlock(&gioc_lock); 192 return 0; 193 } 194 module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug, 195 param_get_int, &mpt3sas_fwfault_debug, 0644); 196 197 /** 198 * _base_readl_aero - retry readl for max three times. 199 * @addr: MPT Fusion system interface register address 200 * 201 * Retry the readl() for max three times if it gets zero value 202 * while reading the system interface register. 203 */ 204 static inline u32 205 _base_readl_aero(const volatile void __iomem *addr) 206 { 207 u32 i = 0, ret_val; 208 209 do { 210 ret_val = readl(addr); 211 i++; 212 } while (ret_val == 0 && i < 3); 213 214 return ret_val; 215 } 216 217 static inline u32 218 _base_readl(const volatile void __iomem *addr) 219 { 220 return readl(addr); 221 } 222 223 /** 224 * _base_clone_reply_to_sys_mem - copies reply to reply free iomem 225 * in BAR0 space. 226 * 227 * @ioc: per adapter object 228 * @reply: reply message frame(lower 32bit addr) 229 * @index: System request message index. 230 */ 231 static void 232 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply, 233 u32 index) 234 { 235 /* 236 * 256 is offset within sys register. 237 * 256 offset MPI frame starts. Max MPI frame supported is 32. 238 * 32 * 128 = 4K. From here, Clone of reply free for mcpu starts 239 */ 240 u16 cmd_credit = ioc->facts.RequestCredit + 1; 241 void __iomem *reply_free_iomem = (void __iomem *)ioc->chip + 242 MPI_FRAME_START_OFFSET + 243 (cmd_credit * ioc->request_sz) + (index * sizeof(u32)); 244 245 writel(reply, reply_free_iomem); 246 } 247 248 /** 249 * _base_clone_mpi_to_sys_mem - Writes/copies MPI frames 250 * to system/BAR0 region. 251 * 252 * @dst_iomem: Pointer to the destination location in BAR0 space. 253 * @src: Pointer to the Source data. 254 * @size: Size of data to be copied. 255 */ 256 static void 257 _base_clone_mpi_to_sys_mem(void *dst_iomem, void *src, u32 size) 258 { 259 int i; 260 u32 *src_virt_mem = (u32 *)src; 261 262 for (i = 0; i < size/4; i++) 263 writel((u32)src_virt_mem[i], 264 (void __iomem *)dst_iomem + (i * 4)); 265 } 266 267 /** 268 * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region 269 * 270 * @dst_iomem: Pointer to the destination location in BAR0 space. 271 * @src: Pointer to the Source data. 272 * @size: Size of data to be copied. 273 */ 274 static void 275 _base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size) 276 { 277 int i; 278 u32 *src_virt_mem = (u32 *)(src); 279 280 for (i = 0; i < size/4; i++) 281 writel((u32)src_virt_mem[i], 282 (void __iomem *)dst_iomem + (i * 4)); 283 } 284 285 /** 286 * _base_get_chain - Calculates and Returns virtual chain address 287 * for the provided smid in BAR0 space. 288 * 289 * @ioc: per adapter object 290 * @smid: system request message index 291 * @sge_chain_count: Scatter gather chain count. 292 * 293 * Return: the chain address. 294 */ 295 static inline void __iomem* 296 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid, 297 u8 sge_chain_count) 298 { 299 void __iomem *base_chain, *chain_virt; 300 u16 cmd_credit = ioc->facts.RequestCredit + 1; 301 302 base_chain = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET + 303 (cmd_credit * ioc->request_sz) + 304 REPLY_FREE_POOL_SIZE; 305 chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth * 306 ioc->request_sz) + (sge_chain_count * ioc->request_sz); 307 return chain_virt; 308 } 309 310 /** 311 * _base_get_chain_phys - Calculates and Returns physical address 312 * in BAR0 for scatter gather chains, for 313 * the provided smid. 314 * 315 * @ioc: per adapter object 316 * @smid: system request message index 317 * @sge_chain_count: Scatter gather chain count. 318 * 319 * Return: Physical chain address. 320 */ 321 static inline phys_addr_t 322 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid, 323 u8 sge_chain_count) 324 { 325 phys_addr_t base_chain_phys, chain_phys; 326 u16 cmd_credit = ioc->facts.RequestCredit + 1; 327 328 base_chain_phys = ioc->chip_phys + MPI_FRAME_START_OFFSET + 329 (cmd_credit * ioc->request_sz) + 330 REPLY_FREE_POOL_SIZE; 331 chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth * 332 ioc->request_sz) + (sge_chain_count * ioc->request_sz); 333 return chain_phys; 334 } 335 336 /** 337 * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host 338 * buffer address for the provided smid. 339 * (Each smid can have 64K starts from 17024) 340 * 341 * @ioc: per adapter object 342 * @smid: system request message index 343 * 344 * Return: Pointer to buffer location in BAR0. 345 */ 346 347 static void __iomem * 348 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid) 349 { 350 u16 cmd_credit = ioc->facts.RequestCredit + 1; 351 // Added extra 1 to reach end of chain. 352 void __iomem *chain_end = _base_get_chain(ioc, 353 cmd_credit + 1, 354 ioc->facts.MaxChainDepth); 355 return chain_end + (smid * 64 * 1024); 356 } 357 358 /** 359 * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped 360 * Host buffer Physical address for the provided smid. 361 * (Each smid can have 64K starts from 17024) 362 * 363 * @ioc: per adapter object 364 * @smid: system request message index 365 * 366 * Return: Pointer to buffer location in BAR0. 367 */ 368 static phys_addr_t 369 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid) 370 { 371 u16 cmd_credit = ioc->facts.RequestCredit + 1; 372 phys_addr_t chain_end_phys = _base_get_chain_phys(ioc, 373 cmd_credit + 1, 374 ioc->facts.MaxChainDepth); 375 return chain_end_phys + (smid * 64 * 1024); 376 } 377 378 /** 379 * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain 380 * lookup list and Provides chain_buffer 381 * address for the matching dma address. 382 * (Each smid can have 64K starts from 17024) 383 * 384 * @ioc: per adapter object 385 * @chain_buffer_dma: Chain buffer dma address. 386 * 387 * Return: Pointer to chain buffer. Or Null on Failure. 388 */ 389 static void * 390 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc, 391 dma_addr_t chain_buffer_dma) 392 { 393 u16 index, j; 394 struct chain_tracker *ct; 395 396 for (index = 0; index < ioc->scsiio_depth; index++) { 397 for (j = 0; j < ioc->chains_needed_per_io; j++) { 398 ct = &ioc->chain_lookup[index].chains_per_smid[j]; 399 if (ct && ct->chain_buffer_dma == chain_buffer_dma) 400 return ct->chain_buffer; 401 } 402 } 403 ioc_info(ioc, "Provided chain_buffer_dma address is not in the lookup list\n"); 404 return NULL; 405 } 406 407 /** 408 * _clone_sg_entries - MPI EP's scsiio and config requests 409 * are handled here. Base function for 410 * double buffering, before submitting 411 * the requests. 412 * 413 * @ioc: per adapter object. 414 * @mpi_request: mf request pointer. 415 * @smid: system request message index. 416 */ 417 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc, 418 void *mpi_request, u16 smid) 419 { 420 Mpi2SGESimple32_t *sgel, *sgel_next; 421 u32 sgl_flags, sge_chain_count = 0; 422 bool is_write = false; 423 u16 i = 0; 424 void __iomem *buffer_iomem; 425 phys_addr_t buffer_iomem_phys; 426 void __iomem *buff_ptr; 427 phys_addr_t buff_ptr_phys; 428 void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO]; 429 void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO]; 430 phys_addr_t dst_addr_phys; 431 MPI2RequestHeader_t *request_hdr; 432 struct scsi_cmnd *scmd; 433 struct scatterlist *sg_scmd = NULL; 434 int is_scsiio_req = 0; 435 436 request_hdr = (MPI2RequestHeader_t *) mpi_request; 437 438 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) { 439 Mpi25SCSIIORequest_t *scsiio_request = 440 (Mpi25SCSIIORequest_t *)mpi_request; 441 sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL; 442 is_scsiio_req = 1; 443 } else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) { 444 Mpi2ConfigRequest_t *config_req = 445 (Mpi2ConfigRequest_t *)mpi_request; 446 sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE; 447 } else 448 return; 449 450 /* From smid we can get scsi_cmd, once we have sg_scmd, 451 * we just need to get sg_virt and sg_next to get virtual 452 * address associated with sgel->Address. 453 */ 454 455 if (is_scsiio_req) { 456 /* Get scsi_cmd using smid */ 457 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid); 458 if (scmd == NULL) { 459 ioc_err(ioc, "scmd is NULL\n"); 460 return; 461 } 462 463 /* Get sg_scmd from scmd provided */ 464 sg_scmd = scsi_sglist(scmd); 465 } 466 467 /* 468 * 0 - 255 System register 469 * 256 - 4352 MPI Frame. (This is based on maxCredit 32) 470 * 4352 - 4864 Reply_free pool (512 byte is reserved 471 * considering maxCredit 32. Reply need extra 472 * room, for mCPU case kept four times of 473 * maxCredit). 474 * 4864 - 17152 SGE chain element. (32cmd * 3 chain of 475 * 128 byte size = 12288) 476 * 17152 - x Host buffer mapped with smid. 477 * (Each smid can have 64K Max IO.) 478 * BAR0+Last 1K MSIX Addr and Data 479 * Total size in use 2113664 bytes of 4MB BAR0 480 */ 481 482 buffer_iomem = _base_get_buffer_bar0(ioc, smid); 483 buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid); 484 485 buff_ptr = buffer_iomem; 486 buff_ptr_phys = buffer_iomem_phys; 487 WARN_ON(buff_ptr_phys > U32_MAX); 488 489 if (le32_to_cpu(sgel->FlagsLength) & 490 (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT)) 491 is_write = true; 492 493 for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) { 494 495 sgl_flags = 496 (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT); 497 498 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) { 499 case MPI2_SGE_FLAGS_CHAIN_ELEMENT: 500 /* 501 * Helper function which on passing 502 * chain_buffer_dma returns chain_buffer. Get 503 * the virtual address for sgel->Address 504 */ 505 sgel_next = 506 _base_get_chain_buffer_dma_to_chain_buffer(ioc, 507 le32_to_cpu(sgel->Address)); 508 if (sgel_next == NULL) 509 return; 510 /* 511 * This is coping 128 byte chain 512 * frame (not a host buffer) 513 */ 514 dst_chain_addr[sge_chain_count] = 515 _base_get_chain(ioc, 516 smid, sge_chain_count); 517 src_chain_addr[sge_chain_count] = 518 (void *) sgel_next; 519 dst_addr_phys = _base_get_chain_phys(ioc, 520 smid, sge_chain_count); 521 WARN_ON(dst_addr_phys > U32_MAX); 522 sgel->Address = 523 cpu_to_le32(lower_32_bits(dst_addr_phys)); 524 sgel = sgel_next; 525 sge_chain_count++; 526 break; 527 case MPI2_SGE_FLAGS_SIMPLE_ELEMENT: 528 if (is_write) { 529 if (is_scsiio_req) { 530 _base_clone_to_sys_mem(buff_ptr, 531 sg_virt(sg_scmd), 532 (le32_to_cpu(sgel->FlagsLength) & 533 0x00ffffff)); 534 /* 535 * FIXME: this relies on a a zero 536 * PCI mem_offset. 537 */ 538 sgel->Address = 539 cpu_to_le32((u32)buff_ptr_phys); 540 } else { 541 _base_clone_to_sys_mem(buff_ptr, 542 ioc->config_vaddr, 543 (le32_to_cpu(sgel->FlagsLength) & 544 0x00ffffff)); 545 sgel->Address = 546 cpu_to_le32((u32)buff_ptr_phys); 547 } 548 } 549 buff_ptr += (le32_to_cpu(sgel->FlagsLength) & 550 0x00ffffff); 551 buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) & 552 0x00ffffff); 553 if ((le32_to_cpu(sgel->FlagsLength) & 554 (MPI2_SGE_FLAGS_END_OF_BUFFER 555 << MPI2_SGE_FLAGS_SHIFT))) 556 goto eob_clone_chain; 557 else { 558 /* 559 * Every single element in MPT will have 560 * associated sg_next. Better to sanity that 561 * sg_next is not NULL, but it will be a bug 562 * if it is null. 563 */ 564 if (is_scsiio_req) { 565 sg_scmd = sg_next(sg_scmd); 566 if (sg_scmd) 567 sgel++; 568 else 569 goto eob_clone_chain; 570 } 571 } 572 break; 573 } 574 } 575 576 eob_clone_chain: 577 for (i = 0; i < sge_chain_count; i++) { 578 if (is_scsiio_req) 579 _base_clone_to_sys_mem(dst_chain_addr[i], 580 src_chain_addr[i], ioc->request_sz); 581 } 582 } 583 584 /** 585 * mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc 586 * @arg: input argument, used to derive ioc 587 * 588 * Return: 589 * 0 if controller is removed from pci subsystem. 590 * -1 for other case. 591 */ 592 static int mpt3sas_remove_dead_ioc_func(void *arg) 593 { 594 struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg; 595 struct pci_dev *pdev; 596 597 if (!ioc) 598 return -1; 599 600 pdev = ioc->pdev; 601 if (!pdev) 602 return -1; 603 pci_stop_and_remove_bus_device_locked(pdev); 604 return 0; 605 } 606 607 /** 608 * _base_sync_drv_fw_timestamp - Sync Drive-Fw TimeStamp. 609 * @ioc: Per Adapter Object 610 * 611 * Return: nothing. 612 */ 613 static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc) 614 { 615 Mpi26IoUnitControlRequest_t *mpi_request; 616 Mpi26IoUnitControlReply_t *mpi_reply; 617 u16 smid; 618 ktime_t current_time; 619 u64 TimeStamp = 0; 620 u8 issue_reset = 0; 621 622 mutex_lock(&ioc->scsih_cmds.mutex); 623 if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) { 624 ioc_err(ioc, "scsih_cmd in use %s\n", __func__); 625 goto out; 626 } 627 ioc->scsih_cmds.status = MPT3_CMD_PENDING; 628 smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx); 629 if (!smid) { 630 ioc_err(ioc, "Failed obtaining a smid %s\n", __func__); 631 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; 632 goto out; 633 } 634 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 635 ioc->scsih_cmds.smid = smid; 636 memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t)); 637 mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL; 638 mpi_request->Operation = MPI26_CTRL_OP_SET_IOC_PARAMETER; 639 mpi_request->IOCParameter = MPI26_SET_IOC_PARAMETER_SYNC_TIMESTAMP; 640 current_time = ktime_get_real(); 641 TimeStamp = ktime_to_ms(current_time); 642 mpi_request->Reserved7 = cpu_to_le32(TimeStamp & 0xFFFFFFFF); 643 mpi_request->IOCParameterValue = cpu_to_le32(TimeStamp >> 32); 644 init_completion(&ioc->scsih_cmds.done); 645 ioc->put_smid_default(ioc, smid); 646 dinitprintk(ioc, ioc_info(ioc, 647 "Io Unit Control Sync TimeStamp (sending), @time %lld ms\n", 648 TimeStamp)); 649 wait_for_completion_timeout(&ioc->scsih_cmds.done, 650 MPT3SAS_TIMESYNC_TIMEOUT_SECONDS*HZ); 651 if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) { 652 mpt3sas_check_cmd_timeout(ioc, 653 ioc->scsih_cmds.status, mpi_request, 654 sizeof(Mpi2SasIoUnitControlRequest_t)/4, issue_reset); 655 goto issue_host_reset; 656 } 657 if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) { 658 mpi_reply = ioc->scsih_cmds.reply; 659 dinitprintk(ioc, ioc_info(ioc, 660 "Io Unit Control sync timestamp (complete): ioc_status(0x%04x), loginfo(0x%08x)\n", 661 le16_to_cpu(mpi_reply->IOCStatus), 662 le32_to_cpu(mpi_reply->IOCLogInfo))); 663 } 664 issue_host_reset: 665 if (issue_reset) 666 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 667 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; 668 out: 669 mutex_unlock(&ioc->scsih_cmds.mutex); 670 } 671 672 /** 673 * _base_fault_reset_work - workq handling ioc fault conditions 674 * @work: input argument, used to derive ioc 675 * 676 * Context: sleep. 677 */ 678 static void 679 _base_fault_reset_work(struct work_struct *work) 680 { 681 struct MPT3SAS_ADAPTER *ioc = 682 container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work); 683 unsigned long flags; 684 u32 doorbell; 685 int rc; 686 struct task_struct *p; 687 688 689 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 690 if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) || 691 ioc->pci_error_recovery) 692 goto rearm_timer; 693 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 694 695 doorbell = mpt3sas_base_get_iocstate(ioc, 0); 696 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) { 697 ioc_err(ioc, "SAS host is non-operational !!!!\n"); 698 699 /* It may be possible that EEH recovery can resolve some of 700 * pci bus failure issues rather removing the dead ioc function 701 * by considering controller is in a non-operational state. So 702 * here priority is given to the EEH recovery. If it doesn't 703 * not resolve this issue, mpt3sas driver will consider this 704 * controller to non-operational state and remove the dead ioc 705 * function. 706 */ 707 if (ioc->non_operational_loop++ < 5) { 708 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, 709 flags); 710 goto rearm_timer; 711 } 712 713 /* 714 * Call _scsih_flush_pending_cmds callback so that we flush all 715 * pending commands back to OS. This call is required to avoid 716 * deadlock at block layer. Dead IOC will fail to do diag reset, 717 * and this call is safe since dead ioc will never return any 718 * command back from HW. 719 */ 720 mpt3sas_base_pause_mq_polling(ioc); 721 ioc->schedule_dead_ioc_flush_running_cmds(ioc); 722 /* 723 * Set remove_host flag early since kernel thread will 724 * take some time to execute. 725 */ 726 ioc->remove_host = 1; 727 /*Remove the Dead Host */ 728 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc, 729 "%s_dead_ioc_%d", ioc->driver_name, ioc->id); 730 if (IS_ERR(p)) 731 ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread failed !!!!\n", 732 __func__); 733 else 734 ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread success !!!!\n", 735 __func__); 736 return; /* don't rearm timer */ 737 } 738 739 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) { 740 u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ? 741 ioc->manu_pg11.CoreDumpTOSec : 742 MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS; 743 744 timeout /= (FAULT_POLLING_INTERVAL/1000); 745 746 if (ioc->ioc_coredump_loop == 0) { 747 mpt3sas_print_coredump_info(ioc, 748 doorbell & MPI2_DOORBELL_DATA_MASK); 749 /* do not accept any IOs and disable the interrupts */ 750 spin_lock_irqsave( 751 &ioc->ioc_reset_in_progress_lock, flags); 752 ioc->shost_recovery = 1; 753 spin_unlock_irqrestore( 754 &ioc->ioc_reset_in_progress_lock, flags); 755 mpt3sas_base_mask_interrupts(ioc); 756 mpt3sas_base_pause_mq_polling(ioc); 757 _base_clear_outstanding_commands(ioc); 758 } 759 760 ioc_info(ioc, "%s: CoreDump loop %d.", 761 __func__, ioc->ioc_coredump_loop); 762 763 /* Wait until CoreDump completes or times out */ 764 if (ioc->ioc_coredump_loop++ < timeout) { 765 spin_lock_irqsave( 766 &ioc->ioc_reset_in_progress_lock, flags); 767 goto rearm_timer; 768 } 769 } 770 771 if (ioc->ioc_coredump_loop) { 772 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_COREDUMP) 773 ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d", 774 __func__, ioc->ioc_coredump_loop); 775 else 776 ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d", 777 __func__, ioc->ioc_coredump_loop); 778 ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE; 779 } 780 ioc->non_operational_loop = 0; 781 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) { 782 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 783 ioc_warn(ioc, "%s: hard reset: %s\n", 784 __func__, rc == 0 ? "success" : "failed"); 785 doorbell = mpt3sas_base_get_iocstate(ioc, 0); 786 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 787 mpt3sas_print_fault_code(ioc, doorbell & 788 MPI2_DOORBELL_DATA_MASK); 789 } else if ((doorbell & MPI2_IOC_STATE_MASK) == 790 MPI2_IOC_STATE_COREDUMP) 791 mpt3sas_print_coredump_info(ioc, doorbell & 792 MPI2_DOORBELL_DATA_MASK); 793 if (rc && (doorbell & MPI2_IOC_STATE_MASK) != 794 MPI2_IOC_STATE_OPERATIONAL) 795 return; /* don't rearm timer */ 796 } 797 ioc->ioc_coredump_loop = 0; 798 if (ioc->time_sync_interval && 799 ++ioc->timestamp_update_count >= ioc->time_sync_interval) { 800 ioc->timestamp_update_count = 0; 801 _base_sync_drv_fw_timestamp(ioc); 802 } 803 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 804 rearm_timer: 805 if (ioc->fault_reset_work_q) 806 queue_delayed_work(ioc->fault_reset_work_q, 807 &ioc->fault_reset_work, 808 msecs_to_jiffies(FAULT_POLLING_INTERVAL)); 809 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 810 } 811 812 /** 813 * mpt3sas_base_start_watchdog - start the fault_reset_work_q 814 * @ioc: per adapter object 815 * 816 * Context: sleep. 817 */ 818 void 819 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc) 820 { 821 unsigned long flags; 822 823 if (ioc->fault_reset_work_q) 824 return; 825 826 ioc->timestamp_update_count = 0; 827 /* initialize fault polling */ 828 829 INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work); 830 snprintf(ioc->fault_reset_work_q_name, 831 sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status", 832 ioc->driver_name, ioc->id); 833 ioc->fault_reset_work_q = 834 create_singlethread_workqueue(ioc->fault_reset_work_q_name); 835 if (!ioc->fault_reset_work_q) { 836 ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__); 837 return; 838 } 839 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 840 if (ioc->fault_reset_work_q) 841 queue_delayed_work(ioc->fault_reset_work_q, 842 &ioc->fault_reset_work, 843 msecs_to_jiffies(FAULT_POLLING_INTERVAL)); 844 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 845 } 846 847 /** 848 * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q 849 * @ioc: per adapter object 850 * 851 * Context: sleep. 852 */ 853 void 854 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc) 855 { 856 unsigned long flags; 857 struct workqueue_struct *wq; 858 859 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 860 wq = ioc->fault_reset_work_q; 861 ioc->fault_reset_work_q = NULL; 862 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 863 if (wq) { 864 if (!cancel_delayed_work_sync(&ioc->fault_reset_work)) 865 flush_workqueue(wq); 866 destroy_workqueue(wq); 867 } 868 } 869 870 /** 871 * mpt3sas_base_fault_info - verbose translation of firmware FAULT code 872 * @ioc: per adapter object 873 * @fault_code: fault code 874 */ 875 void 876 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code) 877 { 878 ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code); 879 } 880 881 /** 882 * mpt3sas_base_coredump_info - verbose translation of firmware CoreDump state 883 * @ioc: per adapter object 884 * @fault_code: fault code 885 * 886 * Return: nothing. 887 */ 888 void 889 mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code) 890 { 891 ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code); 892 } 893 894 /** 895 * mpt3sas_base_wait_for_coredump_completion - Wait until coredump 896 * completes or times out 897 * @ioc: per adapter object 898 * @caller: caller function name 899 * 900 * Return: 0 for success, non-zero for failure. 901 */ 902 int 903 mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc, 904 const char *caller) 905 { 906 u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ? 907 ioc->manu_pg11.CoreDumpTOSec : 908 MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS; 909 910 int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT, 911 timeout); 912 913 if (ioc_state) 914 ioc_err(ioc, 915 "%s: CoreDump timed out. (ioc_state=0x%x)\n", 916 caller, ioc_state); 917 else 918 ioc_info(ioc, 919 "%s: CoreDump completed. (ioc_state=0x%x)\n", 920 caller, ioc_state); 921 922 return ioc_state; 923 } 924 925 /** 926 * mpt3sas_halt_firmware - halt's mpt controller firmware 927 * @ioc: per adapter object 928 * 929 * For debugging timeout related issues. Writing 0xCOFFEE00 930 * to the doorbell register will halt controller firmware. With 931 * the purpose to stop both driver and firmware, the enduser can 932 * obtain a ring buffer from controller UART. 933 */ 934 void 935 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc) 936 { 937 u32 doorbell; 938 939 if (!ioc->fwfault_debug) 940 return; 941 942 dump_stack(); 943 944 doorbell = ioc->base_readl(&ioc->chip->Doorbell); 945 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 946 mpt3sas_print_fault_code(ioc, doorbell & 947 MPI2_DOORBELL_DATA_MASK); 948 } else if ((doorbell & MPI2_IOC_STATE_MASK) == 949 MPI2_IOC_STATE_COREDUMP) { 950 mpt3sas_print_coredump_info(ioc, doorbell & 951 MPI2_DOORBELL_DATA_MASK); 952 } else { 953 writel(0xC0FFEE00, &ioc->chip->Doorbell); 954 ioc_err(ioc, "Firmware is halted due to command timeout\n"); 955 } 956 957 if (ioc->fwfault_debug == 2) 958 for (;;) 959 ; 960 else 961 panic("panic in %s\n", __func__); 962 } 963 964 /** 965 * _base_sas_ioc_info - verbose translation of the ioc status 966 * @ioc: per adapter object 967 * @mpi_reply: reply mf payload returned from firmware 968 * @request_hdr: request mf 969 */ 970 static void 971 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply, 972 MPI2RequestHeader_t *request_hdr) 973 { 974 u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & 975 MPI2_IOCSTATUS_MASK; 976 char *desc = NULL; 977 u16 frame_sz; 978 char *func_str = NULL; 979 980 /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */ 981 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST || 982 request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH || 983 request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION) 984 return; 985 986 if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE) 987 return; 988 /* 989 * Older Firmware version doesn't support driver trigger pages. 990 * So, skip displaying 'config invalid type' type 991 * of error message. 992 */ 993 if (request_hdr->Function == MPI2_FUNCTION_CONFIG) { 994 Mpi2ConfigRequest_t *rqst = (Mpi2ConfigRequest_t *)request_hdr; 995 996 if ((rqst->ExtPageType == 997 MPI2_CONFIG_EXTPAGETYPE_DRIVER_PERSISTENT_TRIGGER) && 998 !(ioc->logging_level & MPT_DEBUG_CONFIG)) { 999 return; 1000 } 1001 } 1002 1003 switch (ioc_status) { 1004 1005 /**************************************************************************** 1006 * Common IOCStatus values for all replies 1007 ****************************************************************************/ 1008 1009 case MPI2_IOCSTATUS_INVALID_FUNCTION: 1010 desc = "invalid function"; 1011 break; 1012 case MPI2_IOCSTATUS_BUSY: 1013 desc = "busy"; 1014 break; 1015 case MPI2_IOCSTATUS_INVALID_SGL: 1016 desc = "invalid sgl"; 1017 break; 1018 case MPI2_IOCSTATUS_INTERNAL_ERROR: 1019 desc = "internal error"; 1020 break; 1021 case MPI2_IOCSTATUS_INVALID_VPID: 1022 desc = "invalid vpid"; 1023 break; 1024 case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES: 1025 desc = "insufficient resources"; 1026 break; 1027 case MPI2_IOCSTATUS_INSUFFICIENT_POWER: 1028 desc = "insufficient power"; 1029 break; 1030 case MPI2_IOCSTATUS_INVALID_FIELD: 1031 desc = "invalid field"; 1032 break; 1033 case MPI2_IOCSTATUS_INVALID_STATE: 1034 desc = "invalid state"; 1035 break; 1036 case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED: 1037 desc = "op state not supported"; 1038 break; 1039 1040 /**************************************************************************** 1041 * Config IOCStatus values 1042 ****************************************************************************/ 1043 1044 case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION: 1045 desc = "config invalid action"; 1046 break; 1047 case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE: 1048 desc = "config invalid type"; 1049 break; 1050 case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE: 1051 desc = "config invalid page"; 1052 break; 1053 case MPI2_IOCSTATUS_CONFIG_INVALID_DATA: 1054 desc = "config invalid data"; 1055 break; 1056 case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS: 1057 desc = "config no defaults"; 1058 break; 1059 case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT: 1060 desc = "config cant commit"; 1061 break; 1062 1063 /**************************************************************************** 1064 * SCSI IO Reply 1065 ****************************************************************************/ 1066 1067 case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR: 1068 case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE: 1069 case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE: 1070 case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN: 1071 case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN: 1072 case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR: 1073 case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR: 1074 case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED: 1075 case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH: 1076 case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED: 1077 case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED: 1078 case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED: 1079 break; 1080 1081 /**************************************************************************** 1082 * For use by SCSI Initiator and SCSI Target end-to-end data protection 1083 ****************************************************************************/ 1084 1085 case MPI2_IOCSTATUS_EEDP_GUARD_ERROR: 1086 desc = "eedp guard error"; 1087 break; 1088 case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR: 1089 desc = "eedp ref tag error"; 1090 break; 1091 case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR: 1092 desc = "eedp app tag error"; 1093 break; 1094 1095 /**************************************************************************** 1096 * SCSI Target values 1097 ****************************************************************************/ 1098 1099 case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX: 1100 desc = "target invalid io index"; 1101 break; 1102 case MPI2_IOCSTATUS_TARGET_ABORTED: 1103 desc = "target aborted"; 1104 break; 1105 case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE: 1106 desc = "target no conn retryable"; 1107 break; 1108 case MPI2_IOCSTATUS_TARGET_NO_CONNECTION: 1109 desc = "target no connection"; 1110 break; 1111 case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH: 1112 desc = "target xfer count mismatch"; 1113 break; 1114 case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR: 1115 desc = "target data offset error"; 1116 break; 1117 case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA: 1118 desc = "target too much write data"; 1119 break; 1120 case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT: 1121 desc = "target iu too short"; 1122 break; 1123 case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT: 1124 desc = "target ack nak timeout"; 1125 break; 1126 case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED: 1127 desc = "target nak received"; 1128 break; 1129 1130 /**************************************************************************** 1131 * Serial Attached SCSI values 1132 ****************************************************************************/ 1133 1134 case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED: 1135 desc = "smp request failed"; 1136 break; 1137 case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN: 1138 desc = "smp data overrun"; 1139 break; 1140 1141 /**************************************************************************** 1142 * Diagnostic Buffer Post / Diagnostic Release values 1143 ****************************************************************************/ 1144 1145 case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED: 1146 desc = "diagnostic released"; 1147 break; 1148 default: 1149 break; 1150 } 1151 1152 if (!desc) 1153 return; 1154 1155 switch (request_hdr->Function) { 1156 case MPI2_FUNCTION_CONFIG: 1157 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size; 1158 func_str = "config_page"; 1159 break; 1160 case MPI2_FUNCTION_SCSI_TASK_MGMT: 1161 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t); 1162 func_str = "task_mgmt"; 1163 break; 1164 case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL: 1165 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t); 1166 func_str = "sas_iounit_ctl"; 1167 break; 1168 case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR: 1169 frame_sz = sizeof(Mpi2SepRequest_t); 1170 func_str = "enclosure"; 1171 break; 1172 case MPI2_FUNCTION_IOC_INIT: 1173 frame_sz = sizeof(Mpi2IOCInitRequest_t); 1174 func_str = "ioc_init"; 1175 break; 1176 case MPI2_FUNCTION_PORT_ENABLE: 1177 frame_sz = sizeof(Mpi2PortEnableRequest_t); 1178 func_str = "port_enable"; 1179 break; 1180 case MPI2_FUNCTION_SMP_PASSTHROUGH: 1181 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size; 1182 func_str = "smp_passthru"; 1183 break; 1184 case MPI2_FUNCTION_NVME_ENCAPSULATED: 1185 frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) + 1186 ioc->sge_size; 1187 func_str = "nvme_encapsulated"; 1188 break; 1189 default: 1190 frame_sz = 32; 1191 func_str = "unknown"; 1192 break; 1193 } 1194 1195 ioc_warn(ioc, "ioc_status: %s(0x%04x), request(0x%p),(%s)\n", 1196 desc, ioc_status, request_hdr, func_str); 1197 1198 _debug_dump_mf(request_hdr, frame_sz/4); 1199 } 1200 1201 /** 1202 * _base_display_event_data - verbose translation of firmware asyn events 1203 * @ioc: per adapter object 1204 * @mpi_reply: reply mf payload returned from firmware 1205 */ 1206 static void 1207 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc, 1208 Mpi2EventNotificationReply_t *mpi_reply) 1209 { 1210 char *desc = NULL; 1211 u16 event; 1212 1213 if (!(ioc->logging_level & MPT_DEBUG_EVENTS)) 1214 return; 1215 1216 event = le16_to_cpu(mpi_reply->Event); 1217 1218 switch (event) { 1219 case MPI2_EVENT_LOG_DATA: 1220 desc = "Log Data"; 1221 break; 1222 case MPI2_EVENT_STATE_CHANGE: 1223 desc = "Status Change"; 1224 break; 1225 case MPI2_EVENT_HARD_RESET_RECEIVED: 1226 desc = "Hard Reset Received"; 1227 break; 1228 case MPI2_EVENT_EVENT_CHANGE: 1229 desc = "Event Change"; 1230 break; 1231 case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE: 1232 desc = "Device Status Change"; 1233 break; 1234 case MPI2_EVENT_IR_OPERATION_STATUS: 1235 if (!ioc->hide_ir_msg) 1236 desc = "IR Operation Status"; 1237 break; 1238 case MPI2_EVENT_SAS_DISCOVERY: 1239 { 1240 Mpi2EventDataSasDiscovery_t *event_data = 1241 (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData; 1242 ioc_info(ioc, "Discovery: (%s)", 1243 event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ? 1244 "start" : "stop"); 1245 if (event_data->DiscoveryStatus) 1246 pr_cont(" discovery_status(0x%08x)", 1247 le32_to_cpu(event_data->DiscoveryStatus)); 1248 pr_cont("\n"); 1249 return; 1250 } 1251 case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE: 1252 desc = "SAS Broadcast Primitive"; 1253 break; 1254 case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE: 1255 desc = "SAS Init Device Status Change"; 1256 break; 1257 case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW: 1258 desc = "SAS Init Table Overflow"; 1259 break; 1260 case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST: 1261 desc = "SAS Topology Change List"; 1262 break; 1263 case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE: 1264 desc = "SAS Enclosure Device Status Change"; 1265 break; 1266 case MPI2_EVENT_IR_VOLUME: 1267 if (!ioc->hide_ir_msg) 1268 desc = "IR Volume"; 1269 break; 1270 case MPI2_EVENT_IR_PHYSICAL_DISK: 1271 if (!ioc->hide_ir_msg) 1272 desc = "IR Physical Disk"; 1273 break; 1274 case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST: 1275 if (!ioc->hide_ir_msg) 1276 desc = "IR Configuration Change List"; 1277 break; 1278 case MPI2_EVENT_LOG_ENTRY_ADDED: 1279 if (!ioc->hide_ir_msg) 1280 desc = "Log Entry Added"; 1281 break; 1282 case MPI2_EVENT_TEMP_THRESHOLD: 1283 desc = "Temperature Threshold"; 1284 break; 1285 case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION: 1286 desc = "Cable Event"; 1287 break; 1288 case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR: 1289 desc = "SAS Device Discovery Error"; 1290 break; 1291 case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE: 1292 desc = "PCIE Device Status Change"; 1293 break; 1294 case MPI2_EVENT_PCIE_ENUMERATION: 1295 { 1296 Mpi26EventDataPCIeEnumeration_t *event_data = 1297 (Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData; 1298 ioc_info(ioc, "PCIE Enumeration: (%s)", 1299 event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED ? 1300 "start" : "stop"); 1301 if (event_data->EnumerationStatus) 1302 pr_cont("enumeration_status(0x%08x)", 1303 le32_to_cpu(event_data->EnumerationStatus)); 1304 pr_cont("\n"); 1305 return; 1306 } 1307 case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST: 1308 desc = "PCIE Topology Change List"; 1309 break; 1310 } 1311 1312 if (!desc) 1313 return; 1314 1315 ioc_info(ioc, "%s\n", desc); 1316 } 1317 1318 /** 1319 * _base_sas_log_info - verbose translation of firmware log info 1320 * @ioc: per adapter object 1321 * @log_info: log info 1322 */ 1323 static void 1324 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info) 1325 { 1326 union loginfo_type { 1327 u32 loginfo; 1328 struct { 1329 u32 subcode:16; 1330 u32 code:8; 1331 u32 originator:4; 1332 u32 bus_type:4; 1333 } dw; 1334 }; 1335 union loginfo_type sas_loginfo; 1336 char *originator_str = NULL; 1337 1338 sas_loginfo.loginfo = log_info; 1339 if (sas_loginfo.dw.bus_type != 3 /*SAS*/) 1340 return; 1341 1342 /* each nexus loss loginfo */ 1343 if (log_info == 0x31170000) 1344 return; 1345 1346 /* eat the loginfos associated with task aborts */ 1347 if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info == 1348 0x31140000 || log_info == 0x31130000)) 1349 return; 1350 1351 switch (sas_loginfo.dw.originator) { 1352 case 0: 1353 originator_str = "IOP"; 1354 break; 1355 case 1: 1356 originator_str = "PL"; 1357 break; 1358 case 2: 1359 if (!ioc->hide_ir_msg) 1360 originator_str = "IR"; 1361 else 1362 originator_str = "WarpDrive"; 1363 break; 1364 } 1365 1366 ioc_warn(ioc, "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n", 1367 log_info, 1368 originator_str, sas_loginfo.dw.code, sas_loginfo.dw.subcode); 1369 } 1370 1371 /** 1372 * _base_display_reply_info - handle reply descriptors depending on IOC Status 1373 * @ioc: per adapter object 1374 * @smid: system request message index 1375 * @msix_index: MSIX table index supplied by the OS 1376 * @reply: reply message frame (lower 32bit addr) 1377 */ 1378 static void 1379 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, 1380 u32 reply) 1381 { 1382 MPI2DefaultReply_t *mpi_reply; 1383 u16 ioc_status; 1384 u32 loginfo = 0; 1385 1386 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 1387 if (unlikely(!mpi_reply)) { 1388 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n", 1389 __FILE__, __LINE__, __func__); 1390 return; 1391 } 1392 ioc_status = le16_to_cpu(mpi_reply->IOCStatus); 1393 1394 if ((ioc_status & MPI2_IOCSTATUS_MASK) && 1395 (ioc->logging_level & MPT_DEBUG_REPLY)) { 1396 _base_sas_ioc_info(ioc , mpi_reply, 1397 mpt3sas_base_get_msg_frame(ioc, smid)); 1398 } 1399 1400 if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) { 1401 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo); 1402 _base_sas_log_info(ioc, loginfo); 1403 } 1404 1405 if (ioc_status || loginfo) { 1406 ioc_status &= MPI2_IOCSTATUS_MASK; 1407 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo); 1408 } 1409 } 1410 1411 /** 1412 * mpt3sas_base_done - base internal command completion routine 1413 * @ioc: per adapter object 1414 * @smid: system request message index 1415 * @msix_index: MSIX table index supplied by the OS 1416 * @reply: reply message frame(lower 32bit addr) 1417 * 1418 * Return: 1419 * 1 meaning mf should be freed from _base_interrupt 1420 * 0 means the mf is freed from this function. 1421 */ 1422 u8 1423 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, 1424 u32 reply) 1425 { 1426 MPI2DefaultReply_t *mpi_reply; 1427 1428 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 1429 if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK) 1430 return mpt3sas_check_for_pending_internal_cmds(ioc, smid); 1431 1432 if (ioc->base_cmds.status == MPT3_CMD_NOT_USED) 1433 return 1; 1434 1435 ioc->base_cmds.status |= MPT3_CMD_COMPLETE; 1436 if (mpi_reply) { 1437 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID; 1438 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4); 1439 } 1440 ioc->base_cmds.status &= ~MPT3_CMD_PENDING; 1441 1442 complete(&ioc->base_cmds.done); 1443 return 1; 1444 } 1445 1446 /** 1447 * _base_async_event - main callback handler for firmware asyn events 1448 * @ioc: per adapter object 1449 * @msix_index: MSIX table index supplied by the OS 1450 * @reply: reply message frame(lower 32bit addr) 1451 * 1452 * Return: 1453 * 1 meaning mf should be freed from _base_interrupt 1454 * 0 means the mf is freed from this function. 1455 */ 1456 static u8 1457 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply) 1458 { 1459 Mpi2EventNotificationReply_t *mpi_reply; 1460 Mpi2EventAckRequest_t *ack_request; 1461 u16 smid; 1462 struct _event_ack_list *delayed_event_ack; 1463 1464 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 1465 if (!mpi_reply) 1466 return 1; 1467 if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION) 1468 return 1; 1469 1470 _base_display_event_data(ioc, mpi_reply); 1471 1472 if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED)) 1473 goto out; 1474 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 1475 if (!smid) { 1476 delayed_event_ack = kzalloc(sizeof(*delayed_event_ack), 1477 GFP_ATOMIC); 1478 if (!delayed_event_ack) 1479 goto out; 1480 INIT_LIST_HEAD(&delayed_event_ack->list); 1481 delayed_event_ack->Event = mpi_reply->Event; 1482 delayed_event_ack->EventContext = mpi_reply->EventContext; 1483 list_add_tail(&delayed_event_ack->list, 1484 &ioc->delayed_event_ack_list); 1485 dewtprintk(ioc, 1486 ioc_info(ioc, "DELAYED: EVENT ACK: event (0x%04x)\n", 1487 le16_to_cpu(mpi_reply->Event))); 1488 goto out; 1489 } 1490 1491 ack_request = mpt3sas_base_get_msg_frame(ioc, smid); 1492 memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t)); 1493 ack_request->Function = MPI2_FUNCTION_EVENT_ACK; 1494 ack_request->Event = mpi_reply->Event; 1495 ack_request->EventContext = mpi_reply->EventContext; 1496 ack_request->VF_ID = 0; /* TODO */ 1497 ack_request->VP_ID = 0; 1498 ioc->put_smid_default(ioc, smid); 1499 1500 out: 1501 1502 /* scsih callback handler */ 1503 mpt3sas_scsih_event_callback(ioc, msix_index, reply); 1504 1505 /* ctl callback handler */ 1506 mpt3sas_ctl_event_callback(ioc, msix_index, reply); 1507 1508 return 1; 1509 } 1510 1511 static struct scsiio_tracker * 1512 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) 1513 { 1514 struct scsi_cmnd *cmd; 1515 1516 if (WARN_ON(!smid) || 1517 WARN_ON(smid >= ioc->hi_priority_smid)) 1518 return NULL; 1519 1520 cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid); 1521 if (cmd) 1522 return scsi_cmd_priv(cmd); 1523 1524 return NULL; 1525 } 1526 1527 /** 1528 * _base_get_cb_idx - obtain the callback index 1529 * @ioc: per adapter object 1530 * @smid: system request message index 1531 * 1532 * Return: callback index. 1533 */ 1534 static u8 1535 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid) 1536 { 1537 int i; 1538 u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1; 1539 u8 cb_idx = 0xFF; 1540 1541 if (smid < ioc->hi_priority_smid) { 1542 struct scsiio_tracker *st; 1543 1544 if (smid < ctl_smid) { 1545 st = _get_st_from_smid(ioc, smid); 1546 if (st) 1547 cb_idx = st->cb_idx; 1548 } else if (smid == ctl_smid) 1549 cb_idx = ioc->ctl_cb_idx; 1550 } else if (smid < ioc->internal_smid) { 1551 i = smid - ioc->hi_priority_smid; 1552 cb_idx = ioc->hpr_lookup[i].cb_idx; 1553 } else if (smid <= ioc->hba_queue_depth) { 1554 i = smid - ioc->internal_smid; 1555 cb_idx = ioc->internal_lookup[i].cb_idx; 1556 } 1557 return cb_idx; 1558 } 1559 1560 /** 1561 * mpt3sas_base_pause_mq_polling - pause polling on the mq poll queues 1562 * when driver is flushing out the IOs. 1563 * @ioc: per adapter object 1564 * 1565 * Pause polling on the mq poll (io uring) queues when driver is flushing 1566 * out the IOs. Otherwise we may see the race condition of completing the same 1567 * IO from two paths. 1568 * 1569 * Returns nothing. 1570 */ 1571 void 1572 mpt3sas_base_pause_mq_polling(struct MPT3SAS_ADAPTER *ioc) 1573 { 1574 int iopoll_q_count = 1575 ioc->reply_queue_count - ioc->iopoll_q_start_index; 1576 int qid; 1577 1578 for (qid = 0; qid < iopoll_q_count; qid++) 1579 atomic_set(&ioc->io_uring_poll_queues[qid].pause, 1); 1580 1581 /* 1582 * wait for current poll to complete. 1583 */ 1584 for (qid = 0; qid < iopoll_q_count; qid++) { 1585 while (atomic_read(&ioc->io_uring_poll_queues[qid].busy)) 1586 udelay(500); 1587 } 1588 } 1589 1590 /** 1591 * mpt3sas_base_resume_mq_polling - Resume polling on mq poll queues. 1592 * @ioc: per adapter object 1593 * 1594 * Returns nothing. 1595 */ 1596 void 1597 mpt3sas_base_resume_mq_polling(struct MPT3SAS_ADAPTER *ioc) 1598 { 1599 int iopoll_q_count = 1600 ioc->reply_queue_count - ioc->iopoll_q_start_index; 1601 int qid; 1602 1603 for (qid = 0; qid < iopoll_q_count; qid++) 1604 atomic_set(&ioc->io_uring_poll_queues[qid].pause, 0); 1605 } 1606 1607 /** 1608 * mpt3sas_base_mask_interrupts - disable interrupts 1609 * @ioc: per adapter object 1610 * 1611 * Disabling ResetIRQ, Reply and Doorbell Interrupts 1612 */ 1613 void 1614 mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc) 1615 { 1616 u32 him_register; 1617 1618 ioc->mask_interrupts = 1; 1619 him_register = ioc->base_readl(&ioc->chip->HostInterruptMask); 1620 him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK; 1621 writel(him_register, &ioc->chip->HostInterruptMask); 1622 ioc->base_readl(&ioc->chip->HostInterruptMask); 1623 } 1624 1625 /** 1626 * mpt3sas_base_unmask_interrupts - enable interrupts 1627 * @ioc: per adapter object 1628 * 1629 * Enabling only Reply Interrupts 1630 */ 1631 void 1632 mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc) 1633 { 1634 u32 him_register; 1635 1636 him_register = ioc->base_readl(&ioc->chip->HostInterruptMask); 1637 him_register &= ~MPI2_HIM_RIM; 1638 writel(him_register, &ioc->chip->HostInterruptMask); 1639 ioc->mask_interrupts = 0; 1640 } 1641 1642 union reply_descriptor { 1643 u64 word; 1644 struct { 1645 u32 low; 1646 u32 high; 1647 } u; 1648 }; 1649 1650 static u32 base_mod64(u64 dividend, u32 divisor) 1651 { 1652 u32 remainder; 1653 1654 if (!divisor) 1655 pr_err("mpt3sas: DIVISOR is zero, in div fn\n"); 1656 remainder = do_div(dividend, divisor); 1657 return remainder; 1658 } 1659 1660 /** 1661 * _base_process_reply_queue - Process reply descriptors from reply 1662 * descriptor post queue. 1663 * @reply_q: per IRQ's reply queue object. 1664 * 1665 * Return: number of reply descriptors processed from reply 1666 * descriptor queue. 1667 */ 1668 static int 1669 _base_process_reply_queue(struct adapter_reply_queue *reply_q) 1670 { 1671 union reply_descriptor rd; 1672 u64 completed_cmds; 1673 u8 request_descript_type; 1674 u16 smid; 1675 u8 cb_idx; 1676 u32 reply; 1677 u8 msix_index = reply_q->msix_index; 1678 struct MPT3SAS_ADAPTER *ioc = reply_q->ioc; 1679 Mpi2ReplyDescriptorsUnion_t *rpf; 1680 u8 rc; 1681 1682 completed_cmds = 0; 1683 if (!atomic_add_unless(&reply_q->busy, 1, 1)) 1684 return completed_cmds; 1685 1686 rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index]; 1687 request_descript_type = rpf->Default.ReplyFlags 1688 & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; 1689 if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { 1690 atomic_dec(&reply_q->busy); 1691 return completed_cmds; 1692 } 1693 1694 cb_idx = 0xFF; 1695 do { 1696 rd.word = le64_to_cpu(rpf->Words); 1697 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX) 1698 goto out; 1699 reply = 0; 1700 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1); 1701 if (request_descript_type == 1702 MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS || 1703 request_descript_type == 1704 MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS || 1705 request_descript_type == 1706 MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) { 1707 cb_idx = _base_get_cb_idx(ioc, smid); 1708 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) && 1709 (likely(mpt_callbacks[cb_idx] != NULL))) { 1710 rc = mpt_callbacks[cb_idx](ioc, smid, 1711 msix_index, 0); 1712 if (rc) 1713 mpt3sas_base_free_smid(ioc, smid); 1714 } 1715 } else if (request_descript_type == 1716 MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) { 1717 reply = le32_to_cpu( 1718 rpf->AddressReply.ReplyFrameAddress); 1719 if (reply > ioc->reply_dma_max_address || 1720 reply < ioc->reply_dma_min_address) 1721 reply = 0; 1722 if (smid) { 1723 cb_idx = _base_get_cb_idx(ioc, smid); 1724 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) && 1725 (likely(mpt_callbacks[cb_idx] != NULL))) { 1726 rc = mpt_callbacks[cb_idx](ioc, smid, 1727 msix_index, reply); 1728 if (reply) 1729 _base_display_reply_info(ioc, 1730 smid, msix_index, reply); 1731 if (rc) 1732 mpt3sas_base_free_smid(ioc, 1733 smid); 1734 } 1735 } else { 1736 _base_async_event(ioc, msix_index, reply); 1737 } 1738 1739 /* reply free queue handling */ 1740 if (reply) { 1741 ioc->reply_free_host_index = 1742 (ioc->reply_free_host_index == 1743 (ioc->reply_free_queue_depth - 1)) ? 1744 0 : ioc->reply_free_host_index + 1; 1745 ioc->reply_free[ioc->reply_free_host_index] = 1746 cpu_to_le32(reply); 1747 if (ioc->is_mcpu_endpoint) 1748 _base_clone_reply_to_sys_mem(ioc, 1749 reply, 1750 ioc->reply_free_host_index); 1751 writel(ioc->reply_free_host_index, 1752 &ioc->chip->ReplyFreeHostIndex); 1753 } 1754 } 1755 1756 rpf->Words = cpu_to_le64(ULLONG_MAX); 1757 reply_q->reply_post_host_index = 1758 (reply_q->reply_post_host_index == 1759 (ioc->reply_post_queue_depth - 1)) ? 0 : 1760 reply_q->reply_post_host_index + 1; 1761 request_descript_type = 1762 reply_q->reply_post_free[reply_q->reply_post_host_index]. 1763 Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; 1764 completed_cmds++; 1765 /* Update the reply post host index after continuously 1766 * processing the threshold number of Reply Descriptors. 1767 * So that FW can find enough entries to post the Reply 1768 * Descriptors in the reply descriptor post queue. 1769 */ 1770 if (completed_cmds >= ioc->thresh_hold) { 1771 if (ioc->combined_reply_queue) { 1772 writel(reply_q->reply_post_host_index | 1773 ((msix_index & 7) << 1774 MPI2_RPHI_MSIX_INDEX_SHIFT), 1775 ioc->replyPostRegisterIndex[msix_index/8]); 1776 } else { 1777 writel(reply_q->reply_post_host_index | 1778 (msix_index << 1779 MPI2_RPHI_MSIX_INDEX_SHIFT), 1780 &ioc->chip->ReplyPostHostIndex); 1781 } 1782 if (!reply_q->is_iouring_poll_q && 1783 !reply_q->irq_poll_scheduled) { 1784 reply_q->irq_poll_scheduled = true; 1785 irq_poll_sched(&reply_q->irqpoll); 1786 } 1787 atomic_dec(&reply_q->busy); 1788 return completed_cmds; 1789 } 1790 if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) 1791 goto out; 1792 if (!reply_q->reply_post_host_index) 1793 rpf = reply_q->reply_post_free; 1794 else 1795 rpf++; 1796 } while (1); 1797 1798 out: 1799 1800 if (!completed_cmds) { 1801 atomic_dec(&reply_q->busy); 1802 return completed_cmds; 1803 } 1804 1805 if (ioc->is_warpdrive) { 1806 writel(reply_q->reply_post_host_index, 1807 ioc->reply_post_host_index[msix_index]); 1808 atomic_dec(&reply_q->busy); 1809 return completed_cmds; 1810 } 1811 1812 /* Update Reply Post Host Index. 1813 * For those HBA's which support combined reply queue feature 1814 * 1. Get the correct Supplemental Reply Post Host Index Register. 1815 * i.e. (msix_index / 8)th entry from Supplemental Reply Post Host 1816 * Index Register address bank i.e replyPostRegisterIndex[], 1817 * 2. Then update this register with new reply host index value 1818 * in ReplyPostIndex field and the MSIxIndex field with 1819 * msix_index value reduced to a value between 0 and 7, 1820 * using a modulo 8 operation. Since each Supplemental Reply Post 1821 * Host Index Register supports 8 MSI-X vectors. 1822 * 1823 * For other HBA's just update the Reply Post Host Index register with 1824 * new reply host index value in ReplyPostIndex Field and msix_index 1825 * value in MSIxIndex field. 1826 */ 1827 if (ioc->combined_reply_queue) 1828 writel(reply_q->reply_post_host_index | ((msix_index & 7) << 1829 MPI2_RPHI_MSIX_INDEX_SHIFT), 1830 ioc->replyPostRegisterIndex[msix_index/8]); 1831 else 1832 writel(reply_q->reply_post_host_index | (msix_index << 1833 MPI2_RPHI_MSIX_INDEX_SHIFT), 1834 &ioc->chip->ReplyPostHostIndex); 1835 atomic_dec(&reply_q->busy); 1836 return completed_cmds; 1837 } 1838 1839 /** 1840 * mpt3sas_blk_mq_poll - poll the blk mq poll queue 1841 * @shost: Scsi_Host object 1842 * @queue_num: hw ctx queue number 1843 * 1844 * Return number of entries that has been processed from poll queue. 1845 */ 1846 int mpt3sas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) 1847 { 1848 struct MPT3SAS_ADAPTER *ioc = 1849 (struct MPT3SAS_ADAPTER *)shost->hostdata; 1850 struct adapter_reply_queue *reply_q; 1851 int num_entries = 0; 1852 int qid = queue_num - ioc->iopoll_q_start_index; 1853 1854 if (atomic_read(&ioc->io_uring_poll_queues[qid].pause) || 1855 !atomic_add_unless(&ioc->io_uring_poll_queues[qid].busy, 1, 1)) 1856 return 0; 1857 1858 reply_q = ioc->io_uring_poll_queues[qid].reply_q; 1859 1860 num_entries = _base_process_reply_queue(reply_q); 1861 atomic_dec(&ioc->io_uring_poll_queues[qid].busy); 1862 1863 return num_entries; 1864 } 1865 1866 /** 1867 * _base_interrupt - MPT adapter (IOC) specific interrupt handler. 1868 * @irq: irq number (not used) 1869 * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure 1870 * 1871 * Return: IRQ_HANDLED if processed, else IRQ_NONE. 1872 */ 1873 static irqreturn_t 1874 _base_interrupt(int irq, void *bus_id) 1875 { 1876 struct adapter_reply_queue *reply_q = bus_id; 1877 struct MPT3SAS_ADAPTER *ioc = reply_q->ioc; 1878 1879 if (ioc->mask_interrupts) 1880 return IRQ_NONE; 1881 if (reply_q->irq_poll_scheduled) 1882 return IRQ_HANDLED; 1883 return ((_base_process_reply_queue(reply_q) > 0) ? 1884 IRQ_HANDLED : IRQ_NONE); 1885 } 1886 1887 /** 1888 * _base_irqpoll - IRQ poll callback handler 1889 * @irqpoll: irq_poll object 1890 * @budget: irq poll weight 1891 * 1892 * Return: number of reply descriptors processed 1893 */ 1894 static int 1895 _base_irqpoll(struct irq_poll *irqpoll, int budget) 1896 { 1897 struct adapter_reply_queue *reply_q; 1898 int num_entries = 0; 1899 1900 reply_q = container_of(irqpoll, struct adapter_reply_queue, 1901 irqpoll); 1902 if (reply_q->irq_line_enable) { 1903 disable_irq_nosync(reply_q->os_irq); 1904 reply_q->irq_line_enable = false; 1905 } 1906 num_entries = _base_process_reply_queue(reply_q); 1907 if (num_entries < budget) { 1908 irq_poll_complete(irqpoll); 1909 reply_q->irq_poll_scheduled = false; 1910 reply_q->irq_line_enable = true; 1911 enable_irq(reply_q->os_irq); 1912 /* 1913 * Go for one more round of processing the 1914 * reply descriptor post queue in case the HBA 1915 * Firmware has posted some reply descriptors 1916 * while reenabling the IRQ. 1917 */ 1918 _base_process_reply_queue(reply_q); 1919 } 1920 1921 return num_entries; 1922 } 1923 1924 /** 1925 * _base_init_irqpolls - initliaze IRQ polls 1926 * @ioc: per adapter object 1927 * 1928 * Return: nothing 1929 */ 1930 static void 1931 _base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc) 1932 { 1933 struct adapter_reply_queue *reply_q, *next; 1934 1935 if (list_empty(&ioc->reply_queue_list)) 1936 return; 1937 1938 list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { 1939 if (reply_q->is_iouring_poll_q) 1940 continue; 1941 irq_poll_init(&reply_q->irqpoll, 1942 ioc->hba_queue_depth/4, _base_irqpoll); 1943 reply_q->irq_poll_scheduled = false; 1944 reply_q->irq_line_enable = true; 1945 reply_q->os_irq = pci_irq_vector(ioc->pdev, 1946 reply_q->msix_index); 1947 } 1948 } 1949 1950 /** 1951 * _base_is_controller_msix_enabled - is controller support muli-reply queues 1952 * @ioc: per adapter object 1953 * 1954 * Return: Whether or not MSI/X is enabled. 1955 */ 1956 static inline int 1957 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc) 1958 { 1959 return (ioc->facts.IOCCapabilities & 1960 MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable; 1961 } 1962 1963 /** 1964 * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts 1965 * @ioc: per adapter object 1966 * @poll: poll over reply descriptor pools incase interrupt for 1967 * timed-out SCSI command got delayed 1968 * Context: non-ISR context 1969 * 1970 * Called when a Task Management request has completed. 1971 */ 1972 void 1973 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll) 1974 { 1975 struct adapter_reply_queue *reply_q; 1976 1977 /* If MSIX capability is turned off 1978 * then multi-queues are not enabled 1979 */ 1980 if (!_base_is_controller_msix_enabled(ioc)) 1981 return; 1982 1983 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 1984 if (ioc->shost_recovery || ioc->remove_host || 1985 ioc->pci_error_recovery) 1986 return; 1987 /* TMs are on msix_index == 0 */ 1988 if (reply_q->msix_index == 0) 1989 continue; 1990 1991 if (reply_q->is_iouring_poll_q) { 1992 _base_process_reply_queue(reply_q); 1993 continue; 1994 } 1995 1996 synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index)); 1997 if (reply_q->irq_poll_scheduled) { 1998 /* Calling irq_poll_disable will wait for any pending 1999 * callbacks to have completed. 2000 */ 2001 irq_poll_disable(&reply_q->irqpoll); 2002 irq_poll_enable(&reply_q->irqpoll); 2003 /* check how the scheduled poll has ended, 2004 * clean up only if necessary 2005 */ 2006 if (reply_q->irq_poll_scheduled) { 2007 reply_q->irq_poll_scheduled = false; 2008 reply_q->irq_line_enable = true; 2009 enable_irq(reply_q->os_irq); 2010 } 2011 } 2012 } 2013 if (poll) 2014 _base_process_reply_queue(reply_q); 2015 } 2016 2017 /** 2018 * mpt3sas_base_release_callback_handler - clear interrupt callback handler 2019 * @cb_idx: callback index 2020 */ 2021 void 2022 mpt3sas_base_release_callback_handler(u8 cb_idx) 2023 { 2024 mpt_callbacks[cb_idx] = NULL; 2025 } 2026 2027 /** 2028 * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler 2029 * @cb_func: callback function 2030 * 2031 * Return: Index of @cb_func. 2032 */ 2033 u8 2034 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func) 2035 { 2036 u8 cb_idx; 2037 2038 for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--) 2039 if (mpt_callbacks[cb_idx] == NULL) 2040 break; 2041 2042 mpt_callbacks[cb_idx] = cb_func; 2043 return cb_idx; 2044 } 2045 2046 /** 2047 * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler 2048 */ 2049 void 2050 mpt3sas_base_initialize_callback_handler(void) 2051 { 2052 u8 cb_idx; 2053 2054 for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++) 2055 mpt3sas_base_release_callback_handler(cb_idx); 2056 } 2057 2058 2059 /** 2060 * _base_build_zero_len_sge - build zero length sg entry 2061 * @ioc: per adapter object 2062 * @paddr: virtual address for SGE 2063 * 2064 * Create a zero length scatter gather entry to insure the IOCs hardware has 2065 * something to use if the target device goes brain dead and tries 2066 * to send data even when none is asked for. 2067 */ 2068 static void 2069 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr) 2070 { 2071 u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT | 2072 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST | 2073 MPI2_SGE_FLAGS_SIMPLE_ELEMENT) << 2074 MPI2_SGE_FLAGS_SHIFT); 2075 ioc->base_add_sg_single(paddr, flags_length, -1); 2076 } 2077 2078 /** 2079 * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr. 2080 * @paddr: virtual address for SGE 2081 * @flags_length: SGE flags and data transfer length 2082 * @dma_addr: Physical address 2083 */ 2084 static void 2085 _base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr) 2086 { 2087 Mpi2SGESimple32_t *sgel = paddr; 2088 2089 flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING | 2090 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT; 2091 sgel->FlagsLength = cpu_to_le32(flags_length); 2092 sgel->Address = cpu_to_le32(dma_addr); 2093 } 2094 2095 2096 /** 2097 * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr. 2098 * @paddr: virtual address for SGE 2099 * @flags_length: SGE flags and data transfer length 2100 * @dma_addr: Physical address 2101 */ 2102 static void 2103 _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr) 2104 { 2105 Mpi2SGESimple64_t *sgel = paddr; 2106 2107 flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING | 2108 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT; 2109 sgel->FlagsLength = cpu_to_le32(flags_length); 2110 sgel->Address = cpu_to_le64(dma_addr); 2111 } 2112 2113 /** 2114 * _base_get_chain_buffer_tracker - obtain chain tracker 2115 * @ioc: per adapter object 2116 * @scmd: SCSI commands of the IO request 2117 * 2118 * Return: chain tracker from chain_lookup table using key as 2119 * smid and smid's chain_offset. 2120 */ 2121 static struct chain_tracker * 2122 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc, 2123 struct scsi_cmnd *scmd) 2124 { 2125 struct chain_tracker *chain_req; 2126 struct scsiio_tracker *st = scsi_cmd_priv(scmd); 2127 u16 smid = st->smid; 2128 u8 chain_offset = 2129 atomic_read(&ioc->chain_lookup[smid - 1].chain_offset); 2130 2131 if (chain_offset == ioc->chains_needed_per_io) 2132 return NULL; 2133 2134 chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset]; 2135 atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset); 2136 return chain_req; 2137 } 2138 2139 2140 /** 2141 * _base_build_sg - build generic sg 2142 * @ioc: per adapter object 2143 * @psge: virtual address for SGE 2144 * @data_out_dma: physical address for WRITES 2145 * @data_out_sz: data xfer size for WRITES 2146 * @data_in_dma: physical address for READS 2147 * @data_in_sz: data xfer size for READS 2148 */ 2149 static void 2150 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge, 2151 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma, 2152 size_t data_in_sz) 2153 { 2154 u32 sgl_flags; 2155 2156 if (!data_out_sz && !data_in_sz) { 2157 _base_build_zero_len_sge(ioc, psge); 2158 return; 2159 } 2160 2161 if (data_out_sz && data_in_sz) { 2162 /* WRITE sgel first */ 2163 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2164 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC); 2165 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2166 ioc->base_add_sg_single(psge, sgl_flags | 2167 data_out_sz, data_out_dma); 2168 2169 /* incr sgel */ 2170 psge += ioc->sge_size; 2171 2172 /* READ sgel last */ 2173 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2174 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER | 2175 MPI2_SGE_FLAGS_END_OF_LIST); 2176 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2177 ioc->base_add_sg_single(psge, sgl_flags | 2178 data_in_sz, data_in_dma); 2179 } else if (data_out_sz) /* WRITE */ { 2180 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2181 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER | 2182 MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC); 2183 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2184 ioc->base_add_sg_single(psge, sgl_flags | 2185 data_out_sz, data_out_dma); 2186 } else if (data_in_sz) /* READ */ { 2187 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2188 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER | 2189 MPI2_SGE_FLAGS_END_OF_LIST); 2190 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2191 ioc->base_add_sg_single(psge, sgl_flags | 2192 data_in_sz, data_in_dma); 2193 } 2194 } 2195 2196 /* IEEE format sgls */ 2197 2198 /** 2199 * _base_build_nvme_prp - This function is called for NVMe end devices to build 2200 * a native SGL (NVMe PRP). 2201 * @ioc: per adapter object 2202 * @smid: system request message index for getting asscociated SGL 2203 * @nvme_encap_request: the NVMe request msg frame pointer 2204 * @data_out_dma: physical address for WRITES 2205 * @data_out_sz: data xfer size for WRITES 2206 * @data_in_dma: physical address for READS 2207 * @data_in_sz: data xfer size for READS 2208 * 2209 * The native SGL is built starting in the first PRP 2210 * entry of the NVMe message (PRP1). If the data buffer is small enough to be 2211 * described entirely using PRP1, then PRP2 is not used. If needed, PRP2 is 2212 * used to describe a larger data buffer. If the data buffer is too large to 2213 * describe using the two PRP entriess inside the NVMe message, then PRP1 2214 * describes the first data memory segment, and PRP2 contains a pointer to a PRP 2215 * list located elsewhere in memory to describe the remaining data memory 2216 * segments. The PRP list will be contiguous. 2217 * 2218 * The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP 2219 * consists of a list of PRP entries to describe a number of noncontigous 2220 * physical memory segments as a single memory buffer, just as a SGL does. Note 2221 * however, that this function is only used by the IOCTL call, so the memory 2222 * given will be guaranteed to be contiguous. There is no need to translate 2223 * non-contiguous SGL into a PRP in this case. All PRPs will describe 2224 * contiguous space that is one page size each. 2225 * 2226 * Each NVMe message contains two PRP entries. The first (PRP1) either contains 2227 * a PRP list pointer or a PRP element, depending upon the command. PRP2 2228 * contains the second PRP element if the memory being described fits within 2 2229 * PRP entries, or a PRP list pointer if the PRP spans more than two entries. 2230 * 2231 * A PRP list pointer contains the address of a PRP list, structured as a linear 2232 * array of PRP entries. Each PRP entry in this list describes a segment of 2233 * physical memory. 2234 * 2235 * Each 64-bit PRP entry comprises an address and an offset field. The address 2236 * always points at the beginning of a 4KB physical memory page, and the offset 2237 * describes where within that 4KB page the memory segment begins. Only the 2238 * first element in a PRP list may contain a non-zero offset, implying that all 2239 * memory segments following the first begin at the start of a 4KB page. 2240 * 2241 * Each PRP element normally describes 4KB of physical memory, with exceptions 2242 * for the first and last elements in the list. If the memory being described 2243 * by the list begins at a non-zero offset within the first 4KB page, then the 2244 * first PRP element will contain a non-zero offset indicating where the region 2245 * begins within the 4KB page. The last memory segment may end before the end 2246 * of the 4KB segment, depending upon the overall size of the memory being 2247 * described by the PRP list. 2248 * 2249 * Since PRP entries lack any indication of size, the overall data buffer length 2250 * is used to determine where the end of the data memory buffer is located, and 2251 * how many PRP entries are required to describe it. 2252 */ 2253 static void 2254 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid, 2255 Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request, 2256 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma, 2257 size_t data_in_sz) 2258 { 2259 int prp_size = NVME_PRP_SIZE; 2260 __le64 *prp_entry, *prp1_entry, *prp2_entry; 2261 __le64 *prp_page; 2262 dma_addr_t prp_entry_dma, prp_page_dma, dma_addr; 2263 u32 offset, entry_len; 2264 u32 page_mask_result, page_mask; 2265 size_t length; 2266 struct mpt3sas_nvme_cmd *nvme_cmd = 2267 (void *)nvme_encap_request->NVMe_Command; 2268 2269 /* 2270 * Not all commands require a data transfer. If no data, just return 2271 * without constructing any PRP. 2272 */ 2273 if (!data_in_sz && !data_out_sz) 2274 return; 2275 prp1_entry = &nvme_cmd->prp1; 2276 prp2_entry = &nvme_cmd->prp2; 2277 prp_entry = prp1_entry; 2278 /* 2279 * For the PRP entries, use the specially allocated buffer of 2280 * contiguous memory. 2281 */ 2282 prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid); 2283 prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid); 2284 2285 /* 2286 * Check if we are within 1 entry of a page boundary we don't 2287 * want our first entry to be a PRP List entry. 2288 */ 2289 page_mask = ioc->page_size - 1; 2290 page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask; 2291 if (!page_mask_result) { 2292 /* Bump up to next page boundary. */ 2293 prp_page = (__le64 *)((u8 *)prp_page + prp_size); 2294 prp_page_dma = prp_page_dma + prp_size; 2295 } 2296 2297 /* 2298 * Set PRP physical pointer, which initially points to the current PRP 2299 * DMA memory page. 2300 */ 2301 prp_entry_dma = prp_page_dma; 2302 2303 /* Get physical address and length of the data buffer. */ 2304 if (data_in_sz) { 2305 dma_addr = data_in_dma; 2306 length = data_in_sz; 2307 } else { 2308 dma_addr = data_out_dma; 2309 length = data_out_sz; 2310 } 2311 2312 /* Loop while the length is not zero. */ 2313 while (length) { 2314 /* 2315 * Check if we need to put a list pointer here if we are at 2316 * page boundary - prp_size (8 bytes). 2317 */ 2318 page_mask_result = (prp_entry_dma + prp_size) & page_mask; 2319 if (!page_mask_result) { 2320 /* 2321 * This is the last entry in a PRP List, so we need to 2322 * put a PRP list pointer here. What this does is: 2323 * - bump the current memory pointer to the next 2324 * address, which will be the next full page. 2325 * - set the PRP Entry to point to that page. This 2326 * is now the PRP List pointer. 2327 * - bump the PRP Entry pointer the start of the 2328 * next page. Since all of this PRP memory is 2329 * contiguous, no need to get a new page - it's 2330 * just the next address. 2331 */ 2332 prp_entry_dma++; 2333 *prp_entry = cpu_to_le64(prp_entry_dma); 2334 prp_entry++; 2335 } 2336 2337 /* Need to handle if entry will be part of a page. */ 2338 offset = dma_addr & page_mask; 2339 entry_len = ioc->page_size - offset; 2340 2341 if (prp_entry == prp1_entry) { 2342 /* 2343 * Must fill in the first PRP pointer (PRP1) before 2344 * moving on. 2345 */ 2346 *prp1_entry = cpu_to_le64(dma_addr); 2347 2348 /* 2349 * Now point to the second PRP entry within the 2350 * command (PRP2). 2351 */ 2352 prp_entry = prp2_entry; 2353 } else if (prp_entry == prp2_entry) { 2354 /* 2355 * Should the PRP2 entry be a PRP List pointer or just 2356 * a regular PRP pointer? If there is more than one 2357 * more page of data, must use a PRP List pointer. 2358 */ 2359 if (length > ioc->page_size) { 2360 /* 2361 * PRP2 will contain a PRP List pointer because 2362 * more PRP's are needed with this command. The 2363 * list will start at the beginning of the 2364 * contiguous buffer. 2365 */ 2366 *prp2_entry = cpu_to_le64(prp_entry_dma); 2367 2368 /* 2369 * The next PRP Entry will be the start of the 2370 * first PRP List. 2371 */ 2372 prp_entry = prp_page; 2373 } else { 2374 /* 2375 * After this, the PRP Entries are complete. 2376 * This command uses 2 PRP's and no PRP list. 2377 */ 2378 *prp2_entry = cpu_to_le64(dma_addr); 2379 } 2380 } else { 2381 /* 2382 * Put entry in list and bump the addresses. 2383 * 2384 * After PRP1 and PRP2 are filled in, this will fill in 2385 * all remaining PRP entries in a PRP List, one per 2386 * each time through the loop. 2387 */ 2388 *prp_entry = cpu_to_le64(dma_addr); 2389 prp_entry++; 2390 prp_entry_dma++; 2391 } 2392 2393 /* 2394 * Bump the phys address of the command's data buffer by the 2395 * entry_len. 2396 */ 2397 dma_addr += entry_len; 2398 2399 /* Decrement length accounting for last partial page. */ 2400 if (entry_len > length) 2401 length = 0; 2402 else 2403 length -= entry_len; 2404 } 2405 } 2406 2407 /** 2408 * base_make_prp_nvme - Prepare PRPs (Physical Region Page) - 2409 * SGLs specific to NVMe drives only 2410 * 2411 * @ioc: per adapter object 2412 * @scmd: SCSI command from the mid-layer 2413 * @mpi_request: mpi request 2414 * @smid: msg Index 2415 * @sge_count: scatter gather element count. 2416 * 2417 * Return: true: PRPs are built 2418 * false: IEEE SGLs needs to be built 2419 */ 2420 static void 2421 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc, 2422 struct scsi_cmnd *scmd, 2423 Mpi25SCSIIORequest_t *mpi_request, 2424 u16 smid, int sge_count) 2425 { 2426 int sge_len, num_prp_in_chain = 0; 2427 Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl; 2428 __le64 *curr_buff; 2429 dma_addr_t msg_dma, sge_addr, offset; 2430 u32 page_mask, page_mask_result; 2431 struct scatterlist *sg_scmd; 2432 u32 first_prp_len; 2433 int data_len = scsi_bufflen(scmd); 2434 u32 nvme_pg_size; 2435 2436 nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE); 2437 /* 2438 * Nvme has a very convoluted prp format. One prp is required 2439 * for each page or partial page. Driver need to split up OS sg_list 2440 * entries if it is longer than one page or cross a page 2441 * boundary. Driver also have to insert a PRP list pointer entry as 2442 * the last entry in each physical page of the PRP list. 2443 * 2444 * NOTE: The first PRP "entry" is actually placed in the first 2445 * SGL entry in the main message as IEEE 64 format. The 2nd 2446 * entry in the main message is the chain element, and the rest 2447 * of the PRP entries are built in the contiguous pcie buffer. 2448 */ 2449 page_mask = nvme_pg_size - 1; 2450 2451 /* 2452 * Native SGL is needed. 2453 * Put a chain element in main message frame that points to the first 2454 * chain buffer. 2455 * 2456 * NOTE: The ChainOffset field must be 0 when using a chain pointer to 2457 * a native SGL. 2458 */ 2459 2460 /* Set main message chain element pointer */ 2461 main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL; 2462 /* 2463 * For NVMe the chain element needs to be the 2nd SG entry in the main 2464 * message. 2465 */ 2466 main_chain_element = (Mpi25IeeeSgeChain64_t *) 2467 ((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64)); 2468 2469 /* 2470 * For the PRP entries, use the specially allocated buffer of 2471 * contiguous memory. Normal chain buffers can't be used 2472 * because each chain buffer would need to be the size of an OS 2473 * page (4k). 2474 */ 2475 curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid); 2476 msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid); 2477 2478 main_chain_element->Address = cpu_to_le64(msg_dma); 2479 main_chain_element->NextChainOffset = 0; 2480 main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT | 2481 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR | 2482 MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP; 2483 2484 /* Build first prp, sge need not to be page aligned*/ 2485 ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL; 2486 sg_scmd = scsi_sglist(scmd); 2487 sge_addr = sg_dma_address(sg_scmd); 2488 sge_len = sg_dma_len(sg_scmd); 2489 2490 offset = sge_addr & page_mask; 2491 first_prp_len = nvme_pg_size - offset; 2492 2493 ptr_first_sgl->Address = cpu_to_le64(sge_addr); 2494 ptr_first_sgl->Length = cpu_to_le32(first_prp_len); 2495 2496 data_len -= first_prp_len; 2497 2498 if (sge_len > first_prp_len) { 2499 sge_addr += first_prp_len; 2500 sge_len -= first_prp_len; 2501 } else if (data_len && (sge_len == first_prp_len)) { 2502 sg_scmd = sg_next(sg_scmd); 2503 sge_addr = sg_dma_address(sg_scmd); 2504 sge_len = sg_dma_len(sg_scmd); 2505 } 2506 2507 for (;;) { 2508 offset = sge_addr & page_mask; 2509 2510 /* Put PRP pointer due to page boundary*/ 2511 page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask; 2512 if (unlikely(!page_mask_result)) { 2513 scmd_printk(KERN_NOTICE, 2514 scmd, "page boundary curr_buff: 0x%p\n", 2515 curr_buff); 2516 msg_dma += 8; 2517 *curr_buff = cpu_to_le64(msg_dma); 2518 curr_buff++; 2519 num_prp_in_chain++; 2520 } 2521 2522 *curr_buff = cpu_to_le64(sge_addr); 2523 curr_buff++; 2524 msg_dma += 8; 2525 num_prp_in_chain++; 2526 2527 sge_addr += nvme_pg_size; 2528 sge_len -= nvme_pg_size; 2529 data_len -= nvme_pg_size; 2530 2531 if (data_len <= 0) 2532 break; 2533 2534 if (sge_len > 0) 2535 continue; 2536 2537 sg_scmd = sg_next(sg_scmd); 2538 sge_addr = sg_dma_address(sg_scmd); 2539 sge_len = sg_dma_len(sg_scmd); 2540 } 2541 2542 main_chain_element->Length = 2543 cpu_to_le32(num_prp_in_chain * sizeof(u64)); 2544 return; 2545 } 2546 2547 static bool 2548 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc, 2549 struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count) 2550 { 2551 u32 data_length = 0; 2552 bool build_prp = true; 2553 2554 data_length = scsi_bufflen(scmd); 2555 if (pcie_device && 2556 (mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))) { 2557 build_prp = false; 2558 return build_prp; 2559 } 2560 2561 /* If Datalenth is <= 16K and number of SGE’s entries are <= 2 2562 * we built IEEE SGL 2563 */ 2564 if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2)) 2565 build_prp = false; 2566 2567 return build_prp; 2568 } 2569 2570 /** 2571 * _base_check_pcie_native_sgl - This function is called for PCIe end devices to 2572 * determine if the driver needs to build a native SGL. If so, that native 2573 * SGL is built in the special contiguous buffers allocated especially for 2574 * PCIe SGL creation. If the driver will not build a native SGL, return 2575 * TRUE and a normal IEEE SGL will be built. Currently this routine 2576 * supports NVMe. 2577 * @ioc: per adapter object 2578 * @mpi_request: mf request pointer 2579 * @smid: system request message index 2580 * @scmd: scsi command 2581 * @pcie_device: points to the PCIe device's info 2582 * 2583 * Return: 0 if native SGL was built, 1 if no SGL was built 2584 */ 2585 static int 2586 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc, 2587 Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd, 2588 struct _pcie_device *pcie_device) 2589 { 2590 int sges_left; 2591 2592 /* Get the SG list pointer and info. */ 2593 sges_left = scsi_dma_map(scmd); 2594 if (sges_left < 0) { 2595 sdev_printk(KERN_ERR, scmd->device, 2596 "scsi_dma_map failed: request for %d bytes!\n", 2597 scsi_bufflen(scmd)); 2598 return 1; 2599 } 2600 2601 /* Check if we need to build a native SG list. */ 2602 if (!base_is_prp_possible(ioc, pcie_device, 2603 scmd, sges_left)) { 2604 /* We built a native SG list, just return. */ 2605 goto out; 2606 } 2607 2608 /* 2609 * Build native NVMe PRP. 2610 */ 2611 base_make_prp_nvme(ioc, scmd, mpi_request, 2612 smid, sges_left); 2613 2614 return 0; 2615 out: 2616 scsi_dma_unmap(scmd); 2617 return 1; 2618 } 2619 2620 /** 2621 * _base_add_sg_single_ieee - add sg element for IEEE format 2622 * @paddr: virtual address for SGE 2623 * @flags: SGE flags 2624 * @chain_offset: number of 128 byte elements from start of segment 2625 * @length: data transfer length 2626 * @dma_addr: Physical address 2627 */ 2628 static void 2629 _base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length, 2630 dma_addr_t dma_addr) 2631 { 2632 Mpi25IeeeSgeChain64_t *sgel = paddr; 2633 2634 sgel->Flags = flags; 2635 sgel->NextChainOffset = chain_offset; 2636 sgel->Length = cpu_to_le32(length); 2637 sgel->Address = cpu_to_le64(dma_addr); 2638 } 2639 2640 /** 2641 * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format 2642 * @ioc: per adapter object 2643 * @paddr: virtual address for SGE 2644 * 2645 * Create a zero length scatter gather entry to insure the IOCs hardware has 2646 * something to use if the target device goes brain dead and tries 2647 * to send data even when none is asked for. 2648 */ 2649 static void 2650 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr) 2651 { 2652 u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2653 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR | 2654 MPI25_IEEE_SGE_FLAGS_END_OF_LIST); 2655 2656 _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1); 2657 } 2658 2659 /** 2660 * _base_build_sg_scmd - main sg creation routine 2661 * pcie_device is unused here! 2662 * @ioc: per adapter object 2663 * @scmd: scsi command 2664 * @smid: system request message index 2665 * @unused: unused pcie_device pointer 2666 * Context: none. 2667 * 2668 * The main routine that builds scatter gather table from a given 2669 * scsi request sent via the .queuecommand main handler. 2670 * 2671 * Return: 0 success, anything else error 2672 */ 2673 static int 2674 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc, 2675 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused) 2676 { 2677 Mpi2SCSIIORequest_t *mpi_request; 2678 dma_addr_t chain_dma; 2679 struct scatterlist *sg_scmd; 2680 void *sg_local, *chain; 2681 u32 chain_offset; 2682 u32 chain_length; 2683 u32 chain_flags; 2684 int sges_left; 2685 u32 sges_in_segment; 2686 u32 sgl_flags; 2687 u32 sgl_flags_last_element; 2688 u32 sgl_flags_end_buffer; 2689 struct chain_tracker *chain_req; 2690 2691 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 2692 2693 /* init scatter gather flags */ 2694 sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT; 2695 if (scmd->sc_data_direction == DMA_TO_DEVICE) 2696 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC; 2697 sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT) 2698 << MPI2_SGE_FLAGS_SHIFT; 2699 sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT | 2700 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST) 2701 << MPI2_SGE_FLAGS_SHIFT; 2702 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2703 2704 sg_scmd = scsi_sglist(scmd); 2705 sges_left = scsi_dma_map(scmd); 2706 if (sges_left < 0) { 2707 sdev_printk(KERN_ERR, scmd->device, 2708 "scsi_dma_map failed: request for %d bytes!\n", 2709 scsi_bufflen(scmd)); 2710 return -ENOMEM; 2711 } 2712 2713 sg_local = &mpi_request->SGL; 2714 sges_in_segment = ioc->max_sges_in_main_message; 2715 if (sges_left <= sges_in_segment) 2716 goto fill_in_last_segment; 2717 2718 mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) + 2719 (sges_in_segment * ioc->sge_size))/4; 2720 2721 /* fill in main message segment when there is a chain following */ 2722 while (sges_in_segment) { 2723 if (sges_in_segment == 1) 2724 ioc->base_add_sg_single(sg_local, 2725 sgl_flags_last_element | sg_dma_len(sg_scmd), 2726 sg_dma_address(sg_scmd)); 2727 else 2728 ioc->base_add_sg_single(sg_local, sgl_flags | 2729 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2730 sg_scmd = sg_next(sg_scmd); 2731 sg_local += ioc->sge_size; 2732 sges_left--; 2733 sges_in_segment--; 2734 } 2735 2736 /* initializing the chain flags and pointers */ 2737 chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT; 2738 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2739 if (!chain_req) 2740 return -1; 2741 chain = chain_req->chain_buffer; 2742 chain_dma = chain_req->chain_buffer_dma; 2743 do { 2744 sges_in_segment = (sges_left <= 2745 ioc->max_sges_in_chain_message) ? sges_left : 2746 ioc->max_sges_in_chain_message; 2747 chain_offset = (sges_left == sges_in_segment) ? 2748 0 : (sges_in_segment * ioc->sge_size)/4; 2749 chain_length = sges_in_segment * ioc->sge_size; 2750 if (chain_offset) { 2751 chain_offset = chain_offset << 2752 MPI2_SGE_CHAIN_OFFSET_SHIFT; 2753 chain_length += ioc->sge_size; 2754 } 2755 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset | 2756 chain_length, chain_dma); 2757 sg_local = chain; 2758 if (!chain_offset) 2759 goto fill_in_last_segment; 2760 2761 /* fill in chain segments */ 2762 while (sges_in_segment) { 2763 if (sges_in_segment == 1) 2764 ioc->base_add_sg_single(sg_local, 2765 sgl_flags_last_element | 2766 sg_dma_len(sg_scmd), 2767 sg_dma_address(sg_scmd)); 2768 else 2769 ioc->base_add_sg_single(sg_local, sgl_flags | 2770 sg_dma_len(sg_scmd), 2771 sg_dma_address(sg_scmd)); 2772 sg_scmd = sg_next(sg_scmd); 2773 sg_local += ioc->sge_size; 2774 sges_left--; 2775 sges_in_segment--; 2776 } 2777 2778 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2779 if (!chain_req) 2780 return -1; 2781 chain = chain_req->chain_buffer; 2782 chain_dma = chain_req->chain_buffer_dma; 2783 } while (1); 2784 2785 2786 fill_in_last_segment: 2787 2788 /* fill the last segment */ 2789 while (sges_left) { 2790 if (sges_left == 1) 2791 ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer | 2792 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2793 else 2794 ioc->base_add_sg_single(sg_local, sgl_flags | 2795 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2796 sg_scmd = sg_next(sg_scmd); 2797 sg_local += ioc->sge_size; 2798 sges_left--; 2799 } 2800 2801 return 0; 2802 } 2803 2804 /** 2805 * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format 2806 * @ioc: per adapter object 2807 * @scmd: scsi command 2808 * @smid: system request message index 2809 * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be 2810 * constructed on need. 2811 * Context: none. 2812 * 2813 * The main routine that builds scatter gather table from a given 2814 * scsi request sent via the .queuecommand main handler. 2815 * 2816 * Return: 0 success, anything else error 2817 */ 2818 static int 2819 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc, 2820 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device) 2821 { 2822 Mpi25SCSIIORequest_t *mpi_request; 2823 dma_addr_t chain_dma; 2824 struct scatterlist *sg_scmd; 2825 void *sg_local, *chain; 2826 u32 chain_offset; 2827 u32 chain_length; 2828 int sges_left; 2829 u32 sges_in_segment; 2830 u8 simple_sgl_flags; 2831 u8 simple_sgl_flags_last; 2832 u8 chain_sgl_flags; 2833 struct chain_tracker *chain_req; 2834 2835 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 2836 2837 /* init scatter gather flags */ 2838 simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2839 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2840 simple_sgl_flags_last = simple_sgl_flags | 2841 MPI25_IEEE_SGE_FLAGS_END_OF_LIST; 2842 chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT | 2843 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2844 2845 /* Check if we need to build a native SG list. */ 2846 if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request, 2847 smid, scmd, pcie_device) == 0)) { 2848 /* We built a native SG list, just return. */ 2849 return 0; 2850 } 2851 2852 sg_scmd = scsi_sglist(scmd); 2853 sges_left = scsi_dma_map(scmd); 2854 if (sges_left < 0) { 2855 sdev_printk(KERN_ERR, scmd->device, 2856 "scsi_dma_map failed: request for %d bytes!\n", 2857 scsi_bufflen(scmd)); 2858 return -ENOMEM; 2859 } 2860 2861 sg_local = &mpi_request->SGL; 2862 sges_in_segment = (ioc->request_sz - 2863 offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee; 2864 if (sges_left <= sges_in_segment) 2865 goto fill_in_last_segment; 2866 2867 mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) + 2868 (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee); 2869 2870 /* fill in main message segment when there is a chain following */ 2871 while (sges_in_segment > 1) { 2872 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0, 2873 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2874 sg_scmd = sg_next(sg_scmd); 2875 sg_local += ioc->sge_size_ieee; 2876 sges_left--; 2877 sges_in_segment--; 2878 } 2879 2880 /* initializing the pointers */ 2881 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2882 if (!chain_req) 2883 return -1; 2884 chain = chain_req->chain_buffer; 2885 chain_dma = chain_req->chain_buffer_dma; 2886 do { 2887 sges_in_segment = (sges_left <= 2888 ioc->max_sges_in_chain_message) ? sges_left : 2889 ioc->max_sges_in_chain_message; 2890 chain_offset = (sges_left == sges_in_segment) ? 2891 0 : sges_in_segment; 2892 chain_length = sges_in_segment * ioc->sge_size_ieee; 2893 if (chain_offset) 2894 chain_length += ioc->sge_size_ieee; 2895 _base_add_sg_single_ieee(sg_local, chain_sgl_flags, 2896 chain_offset, chain_length, chain_dma); 2897 2898 sg_local = chain; 2899 if (!chain_offset) 2900 goto fill_in_last_segment; 2901 2902 /* fill in chain segments */ 2903 while (sges_in_segment) { 2904 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0, 2905 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2906 sg_scmd = sg_next(sg_scmd); 2907 sg_local += ioc->sge_size_ieee; 2908 sges_left--; 2909 sges_in_segment--; 2910 } 2911 2912 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2913 if (!chain_req) 2914 return -1; 2915 chain = chain_req->chain_buffer; 2916 chain_dma = chain_req->chain_buffer_dma; 2917 } while (1); 2918 2919 2920 fill_in_last_segment: 2921 2922 /* fill the last segment */ 2923 while (sges_left > 0) { 2924 if (sges_left == 1) 2925 _base_add_sg_single_ieee(sg_local, 2926 simple_sgl_flags_last, 0, sg_dma_len(sg_scmd), 2927 sg_dma_address(sg_scmd)); 2928 else 2929 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0, 2930 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2931 sg_scmd = sg_next(sg_scmd); 2932 sg_local += ioc->sge_size_ieee; 2933 sges_left--; 2934 } 2935 2936 return 0; 2937 } 2938 2939 /** 2940 * _base_build_sg_ieee - build generic sg for IEEE format 2941 * @ioc: per adapter object 2942 * @psge: virtual address for SGE 2943 * @data_out_dma: physical address for WRITES 2944 * @data_out_sz: data xfer size for WRITES 2945 * @data_in_dma: physical address for READS 2946 * @data_in_sz: data xfer size for READS 2947 */ 2948 static void 2949 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge, 2950 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma, 2951 size_t data_in_sz) 2952 { 2953 u8 sgl_flags; 2954 2955 if (!data_out_sz && !data_in_sz) { 2956 _base_build_zero_len_sge_ieee(ioc, psge); 2957 return; 2958 } 2959 2960 if (data_out_sz && data_in_sz) { 2961 /* WRITE sgel first */ 2962 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2963 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2964 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz, 2965 data_out_dma); 2966 2967 /* incr sgel */ 2968 psge += ioc->sge_size_ieee; 2969 2970 /* READ sgel last */ 2971 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST; 2972 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz, 2973 data_in_dma); 2974 } else if (data_out_sz) /* WRITE */ { 2975 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2976 MPI25_IEEE_SGE_FLAGS_END_OF_LIST | 2977 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2978 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz, 2979 data_out_dma); 2980 } else if (data_in_sz) /* READ */ { 2981 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2982 MPI25_IEEE_SGE_FLAGS_END_OF_LIST | 2983 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2984 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz, 2985 data_in_dma); 2986 } 2987 } 2988 2989 #define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10)) 2990 2991 /** 2992 * _base_config_dma_addressing - set dma addressing 2993 * @ioc: per adapter object 2994 * @pdev: PCI device struct 2995 * 2996 * Return: 0 for success, non-zero for failure. 2997 */ 2998 static int 2999 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev) 3000 { 3001 struct sysinfo s; 3002 3003 if (ioc->is_mcpu_endpoint || 3004 sizeof(dma_addr_t) == 4 || ioc->use_32bit_dma || 3005 dma_get_required_mask(&pdev->dev) <= 32) 3006 ioc->dma_mask = 32; 3007 /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */ 3008 else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) 3009 ioc->dma_mask = 63; 3010 else 3011 ioc->dma_mask = 64; 3012 3013 if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)) || 3014 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask))) 3015 return -ENODEV; 3016 3017 if (ioc->dma_mask > 32) { 3018 ioc->base_add_sg_single = &_base_add_sg_single_64; 3019 ioc->sge_size = sizeof(Mpi2SGESimple64_t); 3020 } else { 3021 ioc->base_add_sg_single = &_base_add_sg_single_32; 3022 ioc->sge_size = sizeof(Mpi2SGESimple32_t); 3023 } 3024 3025 si_meminfo(&s); 3026 ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n", 3027 ioc->dma_mask, convert_to_kb(s.totalram)); 3028 3029 return 0; 3030 } 3031 3032 /** 3033 * _base_check_enable_msix - checks MSIX capabable. 3034 * @ioc: per adapter object 3035 * 3036 * Check to see if card is capable of MSIX, and set number 3037 * of available msix vectors 3038 */ 3039 static int 3040 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc) 3041 { 3042 int base; 3043 u16 message_control; 3044 3045 /* Check whether controller SAS2008 B0 controller, 3046 * if it is SAS2008 B0 controller use IO-APIC instead of MSIX 3047 */ 3048 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 && 3049 ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) { 3050 return -EINVAL; 3051 } 3052 3053 base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX); 3054 if (!base) { 3055 dfailprintk(ioc, ioc_info(ioc, "msix not supported\n")); 3056 return -EINVAL; 3057 } 3058 3059 /* get msix vector count */ 3060 /* NUMA_IO not supported for older controllers */ 3061 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 || 3062 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 || 3063 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 || 3064 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 || 3065 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 || 3066 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 || 3067 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2) 3068 ioc->msix_vector_count = 1; 3069 else { 3070 pci_read_config_word(ioc->pdev, base + 2, &message_control); 3071 ioc->msix_vector_count = (message_control & 0x3FF) + 1; 3072 } 3073 dinitprintk(ioc, ioc_info(ioc, "msix is supported, vector_count(%d)\n", 3074 ioc->msix_vector_count)); 3075 return 0; 3076 } 3077 3078 /** 3079 * mpt3sas_base_free_irq - free irq 3080 * @ioc: per adapter object 3081 * 3082 * Freeing respective reply_queue from the list. 3083 */ 3084 void 3085 mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc) 3086 { 3087 struct adapter_reply_queue *reply_q, *next; 3088 3089 if (list_empty(&ioc->reply_queue_list)) 3090 return; 3091 3092 list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { 3093 list_del(&reply_q->list); 3094 if (reply_q->is_iouring_poll_q) { 3095 kfree(reply_q); 3096 continue; 3097 } 3098 3099 if (ioc->smp_affinity_enable) 3100 irq_set_affinity_hint(pci_irq_vector(ioc->pdev, 3101 reply_q->msix_index), NULL); 3102 free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index), 3103 reply_q); 3104 kfree(reply_q); 3105 } 3106 } 3107 3108 /** 3109 * _base_request_irq - request irq 3110 * @ioc: per adapter object 3111 * @index: msix index into vector table 3112 * 3113 * Inserting respective reply_queue into the list. 3114 */ 3115 static int 3116 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index) 3117 { 3118 struct pci_dev *pdev = ioc->pdev; 3119 struct adapter_reply_queue *reply_q; 3120 int r, qid; 3121 3122 reply_q = kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL); 3123 if (!reply_q) { 3124 ioc_err(ioc, "unable to allocate memory %zu!\n", 3125 sizeof(struct adapter_reply_queue)); 3126 return -ENOMEM; 3127 } 3128 reply_q->ioc = ioc; 3129 reply_q->msix_index = index; 3130 3131 atomic_set(&reply_q->busy, 0); 3132 3133 if (index >= ioc->iopoll_q_start_index) { 3134 qid = index - ioc->iopoll_q_start_index; 3135 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-mq-poll%d", 3136 ioc->driver_name, ioc->id, qid); 3137 reply_q->is_iouring_poll_q = 1; 3138 ioc->io_uring_poll_queues[qid].reply_q = reply_q; 3139 goto out; 3140 } 3141 3142 3143 if (ioc->msix_enable) 3144 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d", 3145 ioc->driver_name, ioc->id, index); 3146 else 3147 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d", 3148 ioc->driver_name, ioc->id); 3149 r = request_irq(pci_irq_vector(pdev, index), _base_interrupt, 3150 IRQF_SHARED, reply_q->name, reply_q); 3151 if (r) { 3152 pr_err("%s: unable to allocate interrupt %d!\n", 3153 reply_q->name, pci_irq_vector(pdev, index)); 3154 kfree(reply_q); 3155 return -EBUSY; 3156 } 3157 out: 3158 INIT_LIST_HEAD(&reply_q->list); 3159 list_add_tail(&reply_q->list, &ioc->reply_queue_list); 3160 return 0; 3161 } 3162 3163 /** 3164 * _base_assign_reply_queues - assigning msix index for each cpu 3165 * @ioc: per adapter object 3166 * 3167 * The enduser would need to set the affinity via /proc/irq/#/smp_affinity 3168 * 3169 * It would nice if we could call irq_set_affinity, however it is not 3170 * an exported symbol 3171 */ 3172 static void 3173 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) 3174 { 3175 unsigned int cpu, nr_cpus, nr_msix, index = 0; 3176 struct adapter_reply_queue *reply_q; 3177 int local_numa_node; 3178 int iopoll_q_count = ioc->reply_queue_count - 3179 ioc->iopoll_q_start_index; 3180 3181 if (!_base_is_controller_msix_enabled(ioc)) 3182 return; 3183 3184 if (ioc->msix_load_balance) 3185 return; 3186 3187 memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz); 3188 3189 nr_cpus = num_online_cpus(); 3190 nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count, 3191 ioc->facts.MaxMSIxVectors); 3192 if (!nr_msix) 3193 return; 3194 3195 if (ioc->smp_affinity_enable) { 3196 3197 /* 3198 * set irq affinity to local numa node for those irqs 3199 * corresponding to high iops queues. 3200 */ 3201 if (ioc->high_iops_queues) { 3202 local_numa_node = dev_to_node(&ioc->pdev->dev); 3203 for (index = 0; index < ioc->high_iops_queues; 3204 index++) { 3205 irq_set_affinity_hint(pci_irq_vector(ioc->pdev, 3206 index), cpumask_of_node(local_numa_node)); 3207 } 3208 } 3209 3210 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 3211 const cpumask_t *mask; 3212 3213 if (reply_q->msix_index < ioc->high_iops_queues || 3214 reply_q->msix_index >= ioc->iopoll_q_start_index) 3215 continue; 3216 3217 mask = pci_irq_get_affinity(ioc->pdev, 3218 reply_q->msix_index); 3219 if (!mask) { 3220 ioc_warn(ioc, "no affinity for msi %x\n", 3221 reply_q->msix_index); 3222 goto fall_back; 3223 } 3224 3225 for_each_cpu_and(cpu, mask, cpu_online_mask) { 3226 if (cpu >= ioc->cpu_msix_table_sz) 3227 break; 3228 ioc->cpu_msix_table[cpu] = reply_q->msix_index; 3229 } 3230 } 3231 return; 3232 } 3233 3234 fall_back: 3235 cpu = cpumask_first(cpu_online_mask); 3236 nr_msix -= (ioc->high_iops_queues - iopoll_q_count); 3237 index = 0; 3238 3239 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 3240 unsigned int i, group = nr_cpus / nr_msix; 3241 3242 if (reply_q->msix_index < ioc->high_iops_queues || 3243 reply_q->msix_index >= ioc->iopoll_q_start_index) 3244 continue; 3245 3246 if (cpu >= nr_cpus) 3247 break; 3248 3249 if (index < nr_cpus % nr_msix) 3250 group++; 3251 3252 for (i = 0 ; i < group ; i++) { 3253 ioc->cpu_msix_table[cpu] = reply_q->msix_index; 3254 cpu = cpumask_next(cpu, cpu_online_mask); 3255 } 3256 index++; 3257 } 3258 } 3259 3260 /** 3261 * _base_check_and_enable_high_iops_queues - enable high iops mode 3262 * @ioc: per adapter object 3263 * @hba_msix_vector_count: msix vectors supported by HBA 3264 * 3265 * Enable high iops queues only if 3266 * - HBA is a SEA/AERO controller and 3267 * - MSI-Xs vector supported by the HBA is 128 and 3268 * - total CPU count in the system >=16 and 3269 * - loaded driver with default max_msix_vectors module parameter and 3270 * - system booted in non kdump mode 3271 * 3272 * Return: nothing. 3273 */ 3274 static void 3275 _base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc, 3276 int hba_msix_vector_count) 3277 { 3278 u16 lnksta, speed; 3279 3280 /* 3281 * Disable high iops queues if io uring poll queues are enabled. 3282 */ 3283 if (perf_mode == MPT_PERF_MODE_IOPS || 3284 perf_mode == MPT_PERF_MODE_LATENCY || 3285 ioc->io_uring_poll_queues) { 3286 ioc->high_iops_queues = 0; 3287 return; 3288 } 3289 3290 if (perf_mode == MPT_PERF_MODE_DEFAULT) { 3291 3292 pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta); 3293 speed = lnksta & PCI_EXP_LNKSTA_CLS; 3294 3295 if (speed < 0x4) { 3296 ioc->high_iops_queues = 0; 3297 return; 3298 } 3299 } 3300 3301 if (!reset_devices && ioc->is_aero_ioc && 3302 hba_msix_vector_count == MPT3SAS_GEN35_MAX_MSIX_QUEUES && 3303 num_online_cpus() >= MPT3SAS_HIGH_IOPS_REPLY_QUEUES && 3304 max_msix_vectors == -1) 3305 ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES; 3306 else 3307 ioc->high_iops_queues = 0; 3308 } 3309 3310 /** 3311 * mpt3sas_base_disable_msix - disables msix 3312 * @ioc: per adapter object 3313 * 3314 */ 3315 void 3316 mpt3sas_base_disable_msix(struct MPT3SAS_ADAPTER *ioc) 3317 { 3318 if (!ioc->msix_enable) 3319 return; 3320 pci_free_irq_vectors(ioc->pdev); 3321 ioc->msix_enable = 0; 3322 kfree(ioc->io_uring_poll_queues); 3323 } 3324 3325 /** 3326 * _base_alloc_irq_vectors - allocate msix vectors 3327 * @ioc: per adapter object 3328 * 3329 */ 3330 static int 3331 _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc) 3332 { 3333 int i, irq_flags = PCI_IRQ_MSIX; 3334 struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues }; 3335 struct irq_affinity *descp = &desc; 3336 /* 3337 * Don't allocate msix vectors for poll_queues. 3338 * msix_vectors is always within a range of FW supported reply queue. 3339 */ 3340 int nr_msix_vectors = ioc->iopoll_q_start_index; 3341 3342 3343 if (ioc->smp_affinity_enable) 3344 irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES; 3345 else 3346 descp = NULL; 3347 3348 ioc_info(ioc, " %d %d %d\n", ioc->high_iops_queues, 3349 ioc->reply_queue_count, nr_msix_vectors); 3350 3351 i = pci_alloc_irq_vectors_affinity(ioc->pdev, 3352 ioc->high_iops_queues, 3353 nr_msix_vectors, irq_flags, descp); 3354 3355 return i; 3356 } 3357 3358 /** 3359 * _base_enable_msix - enables msix, failback to io_apic 3360 * @ioc: per adapter object 3361 * 3362 */ 3363 static int 3364 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) 3365 { 3366 int r; 3367 int i, local_max_msix_vectors; 3368 u8 try_msix = 0; 3369 int iopoll_q_count = 0; 3370 3371 ioc->msix_load_balance = false; 3372 3373 if (msix_disable == -1 || msix_disable == 0) 3374 try_msix = 1; 3375 3376 if (!try_msix) 3377 goto try_ioapic; 3378 3379 if (_base_check_enable_msix(ioc) != 0) 3380 goto try_ioapic; 3381 3382 ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count); 3383 pr_info("\t no of cores: %d, max_msix_vectors: %d\n", 3384 ioc->cpu_count, max_msix_vectors); 3385 3386 ioc->reply_queue_count = 3387 min_t(int, ioc->cpu_count, ioc->msix_vector_count); 3388 3389 if (!ioc->rdpq_array_enable && max_msix_vectors == -1) 3390 local_max_msix_vectors = (reset_devices) ? 1 : 8; 3391 else 3392 local_max_msix_vectors = max_msix_vectors; 3393 3394 if (local_max_msix_vectors == 0) 3395 goto try_ioapic; 3396 3397 /* 3398 * Enable msix_load_balance only if combined reply queue mode is 3399 * disabled on SAS3 & above generation HBA devices. 3400 */ 3401 if (!ioc->combined_reply_queue && 3402 ioc->hba_mpi_version_belonged != MPI2_VERSION) { 3403 ioc_info(ioc, 3404 "combined ReplyQueue is off, Enabling msix load balance\n"); 3405 ioc->msix_load_balance = true; 3406 } 3407 3408 /* 3409 * smp affinity setting is not need when msix load balance 3410 * is enabled. 3411 */ 3412 if (ioc->msix_load_balance) 3413 ioc->smp_affinity_enable = 0; 3414 3415 if (!ioc->smp_affinity_enable || ioc->reply_queue_count <= 1) 3416 ioc->shost->host_tagset = 0; 3417 3418 /* 3419 * Enable io uring poll queues only if host_tagset is enabled. 3420 */ 3421 if (ioc->shost->host_tagset) 3422 iopoll_q_count = poll_queues; 3423 3424 if (iopoll_q_count) { 3425 ioc->io_uring_poll_queues = kcalloc(iopoll_q_count, 3426 sizeof(struct io_uring_poll_queue), GFP_KERNEL); 3427 if (!ioc->io_uring_poll_queues) 3428 iopoll_q_count = 0; 3429 } 3430 3431 if (ioc->is_aero_ioc) 3432 _base_check_and_enable_high_iops_queues(ioc, 3433 ioc->msix_vector_count); 3434 3435 /* 3436 * Add high iops queues count to reply queue count if high iops queues 3437 * are enabled. 3438 */ 3439 ioc->reply_queue_count = min_t(int, 3440 ioc->reply_queue_count + ioc->high_iops_queues, 3441 ioc->msix_vector_count); 3442 3443 /* 3444 * Adjust the reply queue count incase reply queue count 3445 * exceeds the user provided MSIx vectors count. 3446 */ 3447 if (local_max_msix_vectors > 0) 3448 ioc->reply_queue_count = min_t(int, local_max_msix_vectors, 3449 ioc->reply_queue_count); 3450 /* 3451 * Add io uring poll queues count to reply queues count 3452 * if io uring is enabled in driver. 3453 */ 3454 if (iopoll_q_count) { 3455 if (ioc->reply_queue_count < (iopoll_q_count + MPT3_MIN_IRQS)) 3456 iopoll_q_count = 0; 3457 ioc->reply_queue_count = min_t(int, 3458 ioc->reply_queue_count + iopoll_q_count, 3459 ioc->msix_vector_count); 3460 } 3461 3462 /* 3463 * Starting index of io uring poll queues in reply queue list. 3464 */ 3465 ioc->iopoll_q_start_index = 3466 ioc->reply_queue_count - iopoll_q_count; 3467 3468 r = _base_alloc_irq_vectors(ioc); 3469 if (r < 0) { 3470 ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r); 3471 goto try_ioapic; 3472 } 3473 3474 /* 3475 * Adjust the reply queue count if the allocated 3476 * MSIx vectors is less then the requested number 3477 * of MSIx vectors. 3478 */ 3479 if (r < ioc->iopoll_q_start_index) { 3480 ioc->reply_queue_count = r + iopoll_q_count; 3481 ioc->iopoll_q_start_index = 3482 ioc->reply_queue_count - iopoll_q_count; 3483 } 3484 3485 ioc->msix_enable = 1; 3486 for (i = 0; i < ioc->reply_queue_count; i++) { 3487 r = _base_request_irq(ioc, i); 3488 if (r) { 3489 mpt3sas_base_free_irq(ioc); 3490 mpt3sas_base_disable_msix(ioc); 3491 goto try_ioapic; 3492 } 3493 } 3494 3495 ioc_info(ioc, "High IOPs queues : %s\n", 3496 ioc->high_iops_queues ? "enabled" : "disabled"); 3497 3498 return 0; 3499 3500 /* failback to io_apic interrupt routing */ 3501 try_ioapic: 3502 ioc->high_iops_queues = 0; 3503 ioc_info(ioc, "High IOPs queues : disabled\n"); 3504 ioc->reply_queue_count = 1; 3505 ioc->iopoll_q_start_index = ioc->reply_queue_count - 0; 3506 r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY); 3507 if (r < 0) { 3508 dfailprintk(ioc, 3509 ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n", 3510 r)); 3511 } else 3512 r = _base_request_irq(ioc, 0); 3513 3514 return r; 3515 } 3516 3517 /** 3518 * mpt3sas_base_unmap_resources - free controller resources 3519 * @ioc: per adapter object 3520 */ 3521 static void 3522 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc) 3523 { 3524 struct pci_dev *pdev = ioc->pdev; 3525 3526 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 3527 3528 mpt3sas_base_free_irq(ioc); 3529 mpt3sas_base_disable_msix(ioc); 3530 3531 kfree(ioc->replyPostRegisterIndex); 3532 ioc->replyPostRegisterIndex = NULL; 3533 3534 3535 if (ioc->chip_phys) { 3536 iounmap(ioc->chip); 3537 ioc->chip_phys = 0; 3538 } 3539 3540 if (pci_is_enabled(pdev)) { 3541 pci_release_selected_regions(ioc->pdev, ioc->bars); 3542 pci_disable_pcie_error_reporting(pdev); 3543 pci_disable_device(pdev); 3544 } 3545 } 3546 3547 static int 3548 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc); 3549 3550 /** 3551 * mpt3sas_base_check_for_fault_and_issue_reset - check if IOC is in fault state 3552 * and if it is in fault state then issue diag reset. 3553 * @ioc: per adapter object 3554 * 3555 * Return: 0 for success, non-zero for failure. 3556 */ 3557 int 3558 mpt3sas_base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc) 3559 { 3560 u32 ioc_state; 3561 int rc = -EFAULT; 3562 3563 dinitprintk(ioc, pr_info("%s\n", __func__)); 3564 if (ioc->pci_error_recovery) 3565 return 0; 3566 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 3567 dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state)); 3568 3569 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 3570 mpt3sas_print_fault_code(ioc, ioc_state & 3571 MPI2_DOORBELL_DATA_MASK); 3572 mpt3sas_base_mask_interrupts(ioc); 3573 rc = _base_diag_reset(ioc); 3574 } else if ((ioc_state & MPI2_IOC_STATE_MASK) == 3575 MPI2_IOC_STATE_COREDUMP) { 3576 mpt3sas_print_coredump_info(ioc, ioc_state & 3577 MPI2_DOORBELL_DATA_MASK); 3578 mpt3sas_base_wait_for_coredump_completion(ioc, __func__); 3579 mpt3sas_base_mask_interrupts(ioc); 3580 rc = _base_diag_reset(ioc); 3581 } 3582 3583 return rc; 3584 } 3585 3586 /** 3587 * mpt3sas_base_map_resources - map in controller resources (io/irq/memap) 3588 * @ioc: per adapter object 3589 * 3590 * Return: 0 for success, non-zero for failure. 3591 */ 3592 int 3593 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) 3594 { 3595 struct pci_dev *pdev = ioc->pdev; 3596 u32 memap_sz; 3597 u32 pio_sz; 3598 int i, r = 0, rc; 3599 u64 pio_chip = 0; 3600 phys_addr_t chip_phys = 0; 3601 struct adapter_reply_queue *reply_q; 3602 int iopoll_q_count = 0; 3603 3604 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 3605 3606 ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM); 3607 if (pci_enable_device_mem(pdev)) { 3608 ioc_warn(ioc, "pci_enable_device_mem: failed\n"); 3609 ioc->bars = 0; 3610 return -ENODEV; 3611 } 3612 3613 3614 if (pci_request_selected_regions(pdev, ioc->bars, 3615 ioc->driver_name)) { 3616 ioc_warn(ioc, "pci_request_selected_regions: failed\n"); 3617 ioc->bars = 0; 3618 r = -ENODEV; 3619 goto out_fail; 3620 } 3621 3622 /* AER (Advanced Error Reporting) hooks */ 3623 pci_enable_pcie_error_reporting(pdev); 3624 3625 pci_set_master(pdev); 3626 3627 3628 if (_base_config_dma_addressing(ioc, pdev) != 0) { 3629 ioc_warn(ioc, "no suitable DMA mask for %s\n", pci_name(pdev)); 3630 r = -ENODEV; 3631 goto out_fail; 3632 } 3633 3634 for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) && 3635 (!memap_sz || !pio_sz); i++) { 3636 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) { 3637 if (pio_sz) 3638 continue; 3639 pio_chip = (u64)pci_resource_start(pdev, i); 3640 pio_sz = pci_resource_len(pdev, i); 3641 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) { 3642 if (memap_sz) 3643 continue; 3644 ioc->chip_phys = pci_resource_start(pdev, i); 3645 chip_phys = ioc->chip_phys; 3646 memap_sz = pci_resource_len(pdev, i); 3647 ioc->chip = ioremap(ioc->chip_phys, memap_sz); 3648 } 3649 } 3650 3651 if (ioc->chip == NULL) { 3652 ioc_err(ioc, 3653 "unable to map adapter memory! or resource not found\n"); 3654 r = -EINVAL; 3655 goto out_fail; 3656 } 3657 3658 mpt3sas_base_mask_interrupts(ioc); 3659 3660 r = _base_get_ioc_facts(ioc); 3661 if (r) { 3662 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 3663 if (rc || (_base_get_ioc_facts(ioc))) 3664 goto out_fail; 3665 } 3666 3667 if (!ioc->rdpq_array_enable_assigned) { 3668 ioc->rdpq_array_enable = ioc->rdpq_array_capable; 3669 ioc->rdpq_array_enable_assigned = 1; 3670 } 3671 3672 r = _base_enable_msix(ioc); 3673 if (r) 3674 goto out_fail; 3675 3676 iopoll_q_count = ioc->reply_queue_count - ioc->iopoll_q_start_index; 3677 for (i = 0; i < iopoll_q_count; i++) { 3678 atomic_set(&ioc->io_uring_poll_queues[i].busy, 0); 3679 atomic_set(&ioc->io_uring_poll_queues[i].pause, 0); 3680 } 3681 3682 if (!ioc->is_driver_loading) 3683 _base_init_irqpolls(ioc); 3684 /* Use the Combined reply queue feature only for SAS3 C0 & higher 3685 * revision HBAs and also only when reply queue count is greater than 8 3686 */ 3687 if (ioc->combined_reply_queue) { 3688 /* Determine the Supplemental Reply Post Host Index Registers 3689 * Addresse. Supplemental Reply Post Host Index Registers 3690 * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and 3691 * each register is at offset bytes of 3692 * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one. 3693 */ 3694 ioc->replyPostRegisterIndex = kcalloc( 3695 ioc->combined_reply_index_count, 3696 sizeof(resource_size_t *), GFP_KERNEL); 3697 if (!ioc->replyPostRegisterIndex) { 3698 ioc_err(ioc, 3699 "allocation for replyPostRegisterIndex failed!\n"); 3700 r = -ENOMEM; 3701 goto out_fail; 3702 } 3703 3704 for (i = 0; i < ioc->combined_reply_index_count; i++) { 3705 ioc->replyPostRegisterIndex[i] = (resource_size_t *) 3706 ((u8 __force *)&ioc->chip->Doorbell + 3707 MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET + 3708 (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET)); 3709 } 3710 } 3711 3712 if (ioc->is_warpdrive) { 3713 ioc->reply_post_host_index[0] = (resource_size_t __iomem *) 3714 &ioc->chip->ReplyPostHostIndex; 3715 3716 for (i = 1; i < ioc->cpu_msix_table_sz; i++) 3717 ioc->reply_post_host_index[i] = 3718 (resource_size_t __iomem *) 3719 ((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1) 3720 * 4))); 3721 } 3722 3723 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 3724 if (reply_q->msix_index >= ioc->iopoll_q_start_index) { 3725 pr_info("%s: enabled: index: %d\n", 3726 reply_q->name, reply_q->msix_index); 3727 continue; 3728 } 3729 3730 pr_info("%s: %s enabled: IRQ %d\n", 3731 reply_q->name, 3732 ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC", 3733 pci_irq_vector(ioc->pdev, reply_q->msix_index)); 3734 } 3735 3736 ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n", 3737 &chip_phys, ioc->chip, memap_sz); 3738 ioc_info(ioc, "ioport(0x%016llx), size(%d)\n", 3739 (unsigned long long)pio_chip, pio_sz); 3740 3741 /* Save PCI configuration state for recovery from PCI AER/EEH errors */ 3742 pci_save_state(pdev); 3743 return 0; 3744 3745 out_fail: 3746 mpt3sas_base_unmap_resources(ioc); 3747 return r; 3748 } 3749 3750 /** 3751 * mpt3sas_base_get_msg_frame - obtain request mf pointer 3752 * @ioc: per adapter object 3753 * @smid: system request message index(smid zero is invalid) 3754 * 3755 * Return: virt pointer to message frame. 3756 */ 3757 void * 3758 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3759 { 3760 return (void *)(ioc->request + (smid * ioc->request_sz)); 3761 } 3762 3763 /** 3764 * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr 3765 * @ioc: per adapter object 3766 * @smid: system request message index 3767 * 3768 * Return: virt pointer to sense buffer. 3769 */ 3770 void * 3771 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3772 { 3773 return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE)); 3774 } 3775 3776 /** 3777 * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr 3778 * @ioc: per adapter object 3779 * @smid: system request message index 3780 * 3781 * Return: phys pointer to the low 32bit address of the sense buffer. 3782 */ 3783 __le32 3784 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3785 { 3786 return cpu_to_le32(ioc->sense_dma + ((smid - 1) * 3787 SCSI_SENSE_BUFFERSIZE)); 3788 } 3789 3790 /** 3791 * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr 3792 * @ioc: per adapter object 3793 * @smid: system request message index 3794 * 3795 * Return: virt pointer to a PCIe SGL. 3796 */ 3797 void * 3798 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3799 { 3800 return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl); 3801 } 3802 3803 /** 3804 * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr 3805 * @ioc: per adapter object 3806 * @smid: system request message index 3807 * 3808 * Return: phys pointer to the address of the PCIe buffer. 3809 */ 3810 dma_addr_t 3811 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3812 { 3813 return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma; 3814 } 3815 3816 /** 3817 * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address 3818 * @ioc: per adapter object 3819 * @phys_addr: lower 32 physical addr of the reply 3820 * 3821 * Converts 32bit lower physical addr into a virt address. 3822 */ 3823 void * 3824 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr) 3825 { 3826 if (!phys_addr) 3827 return NULL; 3828 return ioc->reply + (phys_addr - (u32)ioc->reply_dma); 3829 } 3830 3831 /** 3832 * _base_get_msix_index - get the msix index 3833 * @ioc: per adapter object 3834 * @scmd: scsi_cmnd object 3835 * 3836 * Return: msix index of general reply queues, 3837 * i.e. reply queue on which IO request's reply 3838 * should be posted by the HBA firmware. 3839 */ 3840 static inline u8 3841 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc, 3842 struct scsi_cmnd *scmd) 3843 { 3844 /* Enables reply_queue load balancing */ 3845 if (ioc->msix_load_balance) 3846 return ioc->reply_queue_count ? 3847 base_mod64(atomic64_add_return(1, 3848 &ioc->total_io_cnt), ioc->reply_queue_count) : 0; 3849 3850 if (scmd && ioc->shost->nr_hw_queues > 1) { 3851 u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); 3852 3853 return blk_mq_unique_tag_to_hwq(tag) + 3854 ioc->high_iops_queues; 3855 } 3856 3857 return ioc->cpu_msix_table[raw_smp_processor_id()]; 3858 } 3859 3860 /** 3861 * _base_get_high_iops_msix_index - get the msix index of 3862 * high iops queues 3863 * @ioc: per adapter object 3864 * @scmd: scsi_cmnd object 3865 * 3866 * Return: msix index of high iops reply queues. 3867 * i.e. high iops reply queue on which IO request's 3868 * reply should be posted by the HBA firmware. 3869 */ 3870 static inline u8 3871 _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc, 3872 struct scsi_cmnd *scmd) 3873 { 3874 /** 3875 * Round robin the IO interrupts among the high iops 3876 * reply queues in terms of batch count 16 when outstanding 3877 * IOs on the target device is >=8. 3878 */ 3879 3880 if (scsi_device_busy(scmd->device) > MPT3SAS_DEVICE_HIGH_IOPS_DEPTH) 3881 return base_mod64(( 3882 atomic64_add_return(1, &ioc->high_iops_outstanding) / 3883 MPT3SAS_HIGH_IOPS_BATCH_COUNT), 3884 MPT3SAS_HIGH_IOPS_REPLY_QUEUES); 3885 3886 return _base_get_msix_index(ioc, scmd); 3887 } 3888 3889 /** 3890 * mpt3sas_base_get_smid - obtain a free smid from internal queue 3891 * @ioc: per adapter object 3892 * @cb_idx: callback index 3893 * 3894 * Return: smid (zero is invalid) 3895 */ 3896 u16 3897 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx) 3898 { 3899 unsigned long flags; 3900 struct request_tracker *request; 3901 u16 smid; 3902 3903 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 3904 if (list_empty(&ioc->internal_free_list)) { 3905 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3906 ioc_err(ioc, "%s: smid not available\n", __func__); 3907 return 0; 3908 } 3909 3910 request = list_entry(ioc->internal_free_list.next, 3911 struct request_tracker, tracker_list); 3912 request->cb_idx = cb_idx; 3913 smid = request->smid; 3914 list_del(&request->tracker_list); 3915 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3916 return smid; 3917 } 3918 3919 /** 3920 * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue 3921 * @ioc: per adapter object 3922 * @cb_idx: callback index 3923 * @scmd: pointer to scsi command object 3924 * 3925 * Return: smid (zero is invalid) 3926 */ 3927 u16 3928 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx, 3929 struct scsi_cmnd *scmd) 3930 { 3931 struct scsiio_tracker *request = scsi_cmd_priv(scmd); 3932 u16 smid; 3933 u32 tag, unique_tag; 3934 3935 unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); 3936 tag = blk_mq_unique_tag_to_tag(unique_tag); 3937 3938 /* 3939 * Store hw queue number corresponding to the tag. 3940 * This hw queue number is used later to determine 3941 * the unique_tag using the logic below. This unique_tag 3942 * is used to retrieve the scmd pointer corresponding 3943 * to tag using scsi_host_find_tag() API. 3944 * 3945 * tag = smid - 1; 3946 * unique_tag = ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag; 3947 */ 3948 ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag); 3949 3950 smid = tag + 1; 3951 request->cb_idx = cb_idx; 3952 request->smid = smid; 3953 request->scmd = scmd; 3954 INIT_LIST_HEAD(&request->chain_list); 3955 return smid; 3956 } 3957 3958 /** 3959 * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue 3960 * @ioc: per adapter object 3961 * @cb_idx: callback index 3962 * 3963 * Return: smid (zero is invalid) 3964 */ 3965 u16 3966 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx) 3967 { 3968 unsigned long flags; 3969 struct request_tracker *request; 3970 u16 smid; 3971 3972 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 3973 if (list_empty(&ioc->hpr_free_list)) { 3974 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3975 return 0; 3976 } 3977 3978 request = list_entry(ioc->hpr_free_list.next, 3979 struct request_tracker, tracker_list); 3980 request->cb_idx = cb_idx; 3981 smid = request->smid; 3982 list_del(&request->tracker_list); 3983 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3984 return smid; 3985 } 3986 3987 static void 3988 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc) 3989 { 3990 /* 3991 * See _wait_for_commands_to_complete() call with regards to this code. 3992 */ 3993 if (ioc->shost_recovery && ioc->pending_io_count) { 3994 ioc->pending_io_count = scsi_host_busy(ioc->shost); 3995 if (ioc->pending_io_count == 0) 3996 wake_up(&ioc->reset_wq); 3997 } 3998 } 3999 4000 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc, 4001 struct scsiio_tracker *st) 4002 { 4003 if (WARN_ON(st->smid == 0)) 4004 return; 4005 st->cb_idx = 0xFF; 4006 st->direct_io = 0; 4007 st->scmd = NULL; 4008 atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0); 4009 st->smid = 0; 4010 } 4011 4012 /** 4013 * mpt3sas_base_free_smid - put smid back on free_list 4014 * @ioc: per adapter object 4015 * @smid: system request message index 4016 */ 4017 void 4018 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4019 { 4020 unsigned long flags; 4021 int i; 4022 4023 if (smid < ioc->hi_priority_smid) { 4024 struct scsiio_tracker *st; 4025 void *request; 4026 4027 st = _get_st_from_smid(ioc, smid); 4028 if (!st) { 4029 _base_recovery_check(ioc); 4030 return; 4031 } 4032 4033 /* Clear MPI request frame */ 4034 request = mpt3sas_base_get_msg_frame(ioc, smid); 4035 memset(request, 0, ioc->request_sz); 4036 4037 mpt3sas_base_clear_st(ioc, st); 4038 _base_recovery_check(ioc); 4039 ioc->io_queue_num[smid - 1] = 0; 4040 return; 4041 } 4042 4043 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 4044 if (smid < ioc->internal_smid) { 4045 /* hi-priority */ 4046 i = smid - ioc->hi_priority_smid; 4047 ioc->hpr_lookup[i].cb_idx = 0xFF; 4048 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list); 4049 } else if (smid <= ioc->hba_queue_depth) { 4050 /* internal queue */ 4051 i = smid - ioc->internal_smid; 4052 ioc->internal_lookup[i].cb_idx = 0xFF; 4053 list_add(&ioc->internal_lookup[i].tracker_list, 4054 &ioc->internal_free_list); 4055 } 4056 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 4057 } 4058 4059 /** 4060 * _base_mpi_ep_writeq - 32 bit write to MMIO 4061 * @b: data payload 4062 * @addr: address in MMIO space 4063 * @writeq_lock: spin lock 4064 * 4065 * This special handling for MPI EP to take care of 32 bit 4066 * environment where its not quarenteed to send the entire word 4067 * in one transfer. 4068 */ 4069 static inline void 4070 _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr, 4071 spinlock_t *writeq_lock) 4072 { 4073 unsigned long flags; 4074 4075 spin_lock_irqsave(writeq_lock, flags); 4076 __raw_writel((u32)(b), addr); 4077 __raw_writel((u32)(b >> 32), (addr + 4)); 4078 spin_unlock_irqrestore(writeq_lock, flags); 4079 } 4080 4081 /** 4082 * _base_writeq - 64 bit write to MMIO 4083 * @b: data payload 4084 * @addr: address in MMIO space 4085 * @writeq_lock: spin lock 4086 * 4087 * Glue for handling an atomic 64 bit word to MMIO. This special handling takes 4088 * care of 32 bit environment where its not quarenteed to send the entire word 4089 * in one transfer. 4090 */ 4091 #if defined(writeq) && defined(CONFIG_64BIT) 4092 static inline void 4093 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock) 4094 { 4095 wmb(); 4096 __raw_writeq(b, addr); 4097 barrier(); 4098 } 4099 #else 4100 static inline void 4101 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock) 4102 { 4103 _base_mpi_ep_writeq(b, addr, writeq_lock); 4104 } 4105 #endif 4106 4107 /** 4108 * _base_set_and_get_msix_index - get the msix index and assign to msix_io 4109 * variable of scsi tracker 4110 * @ioc: per adapter object 4111 * @smid: system request message index 4112 * 4113 * Return: msix index. 4114 */ 4115 static u8 4116 _base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4117 { 4118 struct scsiio_tracker *st = NULL; 4119 4120 if (smid < ioc->hi_priority_smid) 4121 st = _get_st_from_smid(ioc, smid); 4122 4123 if (st == NULL) 4124 return _base_get_msix_index(ioc, NULL); 4125 4126 st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd); 4127 return st->msix_io; 4128 } 4129 4130 /** 4131 * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware 4132 * @ioc: per adapter object 4133 * @smid: system request message index 4134 * @handle: device handle 4135 */ 4136 static void 4137 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, 4138 u16 smid, u16 handle) 4139 { 4140 Mpi2RequestDescriptorUnion_t descriptor; 4141 u64 *request = (u64 *)&descriptor; 4142 void *mpi_req_iomem; 4143 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid); 4144 4145 _clone_sg_entries(ioc, (void *) mfp, smid); 4146 mpi_req_iomem = (void __force *)ioc->chip + 4147 MPI_FRAME_START_OFFSET + (smid * ioc->request_sz); 4148 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, 4149 ioc->request_sz); 4150 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; 4151 descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4152 descriptor.SCSIIO.SMID = cpu_to_le16(smid); 4153 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); 4154 descriptor.SCSIIO.LMID = 0; 4155 _base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4156 &ioc->scsi_lookup_lock); 4157 } 4158 4159 /** 4160 * _base_put_smid_scsi_io - send SCSI_IO request to firmware 4161 * @ioc: per adapter object 4162 * @smid: system request message index 4163 * @handle: device handle 4164 */ 4165 static void 4166 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) 4167 { 4168 Mpi2RequestDescriptorUnion_t descriptor; 4169 u64 *request = (u64 *)&descriptor; 4170 4171 4172 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; 4173 descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4174 descriptor.SCSIIO.SMID = cpu_to_le16(smid); 4175 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); 4176 descriptor.SCSIIO.LMID = 0; 4177 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4178 &ioc->scsi_lookup_lock); 4179 } 4180 4181 /** 4182 * _base_put_smid_fast_path - send fast path request to firmware 4183 * @ioc: per adapter object 4184 * @smid: system request message index 4185 * @handle: device handle 4186 */ 4187 static void 4188 _base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4189 u16 handle) 4190 { 4191 Mpi2RequestDescriptorUnion_t descriptor; 4192 u64 *request = (u64 *)&descriptor; 4193 4194 descriptor.SCSIIO.RequestFlags = 4195 MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; 4196 descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4197 descriptor.SCSIIO.SMID = cpu_to_le16(smid); 4198 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); 4199 descriptor.SCSIIO.LMID = 0; 4200 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4201 &ioc->scsi_lookup_lock); 4202 } 4203 4204 /** 4205 * _base_put_smid_hi_priority - send Task Management request to firmware 4206 * @ioc: per adapter object 4207 * @smid: system request message index 4208 * @msix_task: msix_task will be same as msix of IO in case of task abort else 0 4209 */ 4210 static void 4211 _base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4212 u16 msix_task) 4213 { 4214 Mpi2RequestDescriptorUnion_t descriptor; 4215 void *mpi_req_iomem; 4216 u64 *request; 4217 4218 if (ioc->is_mcpu_endpoint) { 4219 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid); 4220 4221 /* TBD 256 is offset within sys register. */ 4222 mpi_req_iomem = (void __force *)ioc->chip 4223 + MPI_FRAME_START_OFFSET 4224 + (smid * ioc->request_sz); 4225 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, 4226 ioc->request_sz); 4227 } 4228 4229 request = (u64 *)&descriptor; 4230 4231 descriptor.HighPriority.RequestFlags = 4232 MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; 4233 descriptor.HighPriority.MSIxIndex = msix_task; 4234 descriptor.HighPriority.SMID = cpu_to_le16(smid); 4235 descriptor.HighPriority.LMID = 0; 4236 descriptor.HighPriority.Reserved1 = 0; 4237 if (ioc->is_mcpu_endpoint) 4238 _base_mpi_ep_writeq(*request, 4239 &ioc->chip->RequestDescriptorPostLow, 4240 &ioc->scsi_lookup_lock); 4241 else 4242 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4243 &ioc->scsi_lookup_lock); 4244 } 4245 4246 /** 4247 * mpt3sas_base_put_smid_nvme_encap - send NVMe encapsulated request to 4248 * firmware 4249 * @ioc: per adapter object 4250 * @smid: system request message index 4251 */ 4252 void 4253 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4254 { 4255 Mpi2RequestDescriptorUnion_t descriptor; 4256 u64 *request = (u64 *)&descriptor; 4257 4258 descriptor.Default.RequestFlags = 4259 MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED; 4260 descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4261 descriptor.Default.SMID = cpu_to_le16(smid); 4262 descriptor.Default.LMID = 0; 4263 descriptor.Default.DescriptorTypeDependent = 0; 4264 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4265 &ioc->scsi_lookup_lock); 4266 } 4267 4268 /** 4269 * _base_put_smid_default - Default, primarily used for config pages 4270 * @ioc: per adapter object 4271 * @smid: system request message index 4272 */ 4273 static void 4274 _base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4275 { 4276 Mpi2RequestDescriptorUnion_t descriptor; 4277 void *mpi_req_iomem; 4278 u64 *request; 4279 4280 if (ioc->is_mcpu_endpoint) { 4281 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid); 4282 4283 _clone_sg_entries(ioc, (void *) mfp, smid); 4284 /* TBD 256 is offset within sys register */ 4285 mpi_req_iomem = (void __force *)ioc->chip + 4286 MPI_FRAME_START_OFFSET + (smid * ioc->request_sz); 4287 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, 4288 ioc->request_sz); 4289 } 4290 request = (u64 *)&descriptor; 4291 descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 4292 descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4293 descriptor.Default.SMID = cpu_to_le16(smid); 4294 descriptor.Default.LMID = 0; 4295 descriptor.Default.DescriptorTypeDependent = 0; 4296 if (ioc->is_mcpu_endpoint) 4297 _base_mpi_ep_writeq(*request, 4298 &ioc->chip->RequestDescriptorPostLow, 4299 &ioc->scsi_lookup_lock); 4300 else 4301 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4302 &ioc->scsi_lookup_lock); 4303 } 4304 4305 /** 4306 * _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using 4307 * Atomic Request Descriptor 4308 * @ioc: per adapter object 4309 * @smid: system request message index 4310 * @handle: device handle, unused in this function, for function type match 4311 * 4312 * Return: nothing. 4313 */ 4314 static void 4315 _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4316 u16 handle) 4317 { 4318 Mpi26AtomicRequestDescriptor_t descriptor; 4319 u32 *request = (u32 *)&descriptor; 4320 4321 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; 4322 descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4323 descriptor.SMID = cpu_to_le16(smid); 4324 4325 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4326 } 4327 4328 /** 4329 * _base_put_smid_fast_path_atomic - send fast path request to firmware 4330 * using Atomic Request Descriptor 4331 * @ioc: per adapter object 4332 * @smid: system request message index 4333 * @handle: device handle, unused in this function, for function type match 4334 * Return: nothing 4335 */ 4336 static void 4337 _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4338 u16 handle) 4339 { 4340 Mpi26AtomicRequestDescriptor_t descriptor; 4341 u32 *request = (u32 *)&descriptor; 4342 4343 descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; 4344 descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4345 descriptor.SMID = cpu_to_le16(smid); 4346 4347 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4348 } 4349 4350 /** 4351 * _base_put_smid_hi_priority_atomic - send Task Management request to 4352 * firmware using Atomic Request Descriptor 4353 * @ioc: per adapter object 4354 * @smid: system request message index 4355 * @msix_task: msix_task will be same as msix of IO in case of task abort else 0 4356 * 4357 * Return: nothing. 4358 */ 4359 static void 4360 _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4361 u16 msix_task) 4362 { 4363 Mpi26AtomicRequestDescriptor_t descriptor; 4364 u32 *request = (u32 *)&descriptor; 4365 4366 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; 4367 descriptor.MSIxIndex = msix_task; 4368 descriptor.SMID = cpu_to_le16(smid); 4369 4370 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4371 } 4372 4373 /** 4374 * _base_put_smid_default_atomic - Default, primarily used for config pages 4375 * use Atomic Request Descriptor 4376 * @ioc: per adapter object 4377 * @smid: system request message index 4378 * 4379 * Return: nothing. 4380 */ 4381 static void 4382 _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4383 { 4384 Mpi26AtomicRequestDescriptor_t descriptor; 4385 u32 *request = (u32 *)&descriptor; 4386 4387 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 4388 descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4389 descriptor.SMID = cpu_to_le16(smid); 4390 4391 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4392 } 4393 4394 /** 4395 * _base_display_OEMs_branding - Display branding string 4396 * @ioc: per adapter object 4397 */ 4398 static void 4399 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) 4400 { 4401 if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL) 4402 return; 4403 4404 switch (ioc->pdev->subsystem_vendor) { 4405 case PCI_VENDOR_ID_INTEL: 4406 switch (ioc->pdev->device) { 4407 case MPI2_MFGPAGE_DEVID_SAS2008: 4408 switch (ioc->pdev->subsystem_device) { 4409 case MPT2SAS_INTEL_RMS2LL080_SSDID: 4410 ioc_info(ioc, "%s\n", 4411 MPT2SAS_INTEL_RMS2LL080_BRANDING); 4412 break; 4413 case MPT2SAS_INTEL_RMS2LL040_SSDID: 4414 ioc_info(ioc, "%s\n", 4415 MPT2SAS_INTEL_RMS2LL040_BRANDING); 4416 break; 4417 case MPT2SAS_INTEL_SSD910_SSDID: 4418 ioc_info(ioc, "%s\n", 4419 MPT2SAS_INTEL_SSD910_BRANDING); 4420 break; 4421 default: 4422 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4423 ioc->pdev->subsystem_device); 4424 break; 4425 } 4426 break; 4427 case MPI2_MFGPAGE_DEVID_SAS2308_2: 4428 switch (ioc->pdev->subsystem_device) { 4429 case MPT2SAS_INTEL_RS25GB008_SSDID: 4430 ioc_info(ioc, "%s\n", 4431 MPT2SAS_INTEL_RS25GB008_BRANDING); 4432 break; 4433 case MPT2SAS_INTEL_RMS25JB080_SSDID: 4434 ioc_info(ioc, "%s\n", 4435 MPT2SAS_INTEL_RMS25JB080_BRANDING); 4436 break; 4437 case MPT2SAS_INTEL_RMS25JB040_SSDID: 4438 ioc_info(ioc, "%s\n", 4439 MPT2SAS_INTEL_RMS25JB040_BRANDING); 4440 break; 4441 case MPT2SAS_INTEL_RMS25KB080_SSDID: 4442 ioc_info(ioc, "%s\n", 4443 MPT2SAS_INTEL_RMS25KB080_BRANDING); 4444 break; 4445 case MPT2SAS_INTEL_RMS25KB040_SSDID: 4446 ioc_info(ioc, "%s\n", 4447 MPT2SAS_INTEL_RMS25KB040_BRANDING); 4448 break; 4449 case MPT2SAS_INTEL_RMS25LB040_SSDID: 4450 ioc_info(ioc, "%s\n", 4451 MPT2SAS_INTEL_RMS25LB040_BRANDING); 4452 break; 4453 case MPT2SAS_INTEL_RMS25LB080_SSDID: 4454 ioc_info(ioc, "%s\n", 4455 MPT2SAS_INTEL_RMS25LB080_BRANDING); 4456 break; 4457 default: 4458 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4459 ioc->pdev->subsystem_device); 4460 break; 4461 } 4462 break; 4463 case MPI25_MFGPAGE_DEVID_SAS3008: 4464 switch (ioc->pdev->subsystem_device) { 4465 case MPT3SAS_INTEL_RMS3JC080_SSDID: 4466 ioc_info(ioc, "%s\n", 4467 MPT3SAS_INTEL_RMS3JC080_BRANDING); 4468 break; 4469 4470 case MPT3SAS_INTEL_RS3GC008_SSDID: 4471 ioc_info(ioc, "%s\n", 4472 MPT3SAS_INTEL_RS3GC008_BRANDING); 4473 break; 4474 case MPT3SAS_INTEL_RS3FC044_SSDID: 4475 ioc_info(ioc, "%s\n", 4476 MPT3SAS_INTEL_RS3FC044_BRANDING); 4477 break; 4478 case MPT3SAS_INTEL_RS3UC080_SSDID: 4479 ioc_info(ioc, "%s\n", 4480 MPT3SAS_INTEL_RS3UC080_BRANDING); 4481 break; 4482 default: 4483 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4484 ioc->pdev->subsystem_device); 4485 break; 4486 } 4487 break; 4488 default: 4489 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4490 ioc->pdev->subsystem_device); 4491 break; 4492 } 4493 break; 4494 case PCI_VENDOR_ID_DELL: 4495 switch (ioc->pdev->device) { 4496 case MPI2_MFGPAGE_DEVID_SAS2008: 4497 switch (ioc->pdev->subsystem_device) { 4498 case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID: 4499 ioc_info(ioc, "%s\n", 4500 MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING); 4501 break; 4502 case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID: 4503 ioc_info(ioc, "%s\n", 4504 MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING); 4505 break; 4506 case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID: 4507 ioc_info(ioc, "%s\n", 4508 MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING); 4509 break; 4510 case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID: 4511 ioc_info(ioc, "%s\n", 4512 MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING); 4513 break; 4514 case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID: 4515 ioc_info(ioc, "%s\n", 4516 MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING); 4517 break; 4518 case MPT2SAS_DELL_PERC_H200_SSDID: 4519 ioc_info(ioc, "%s\n", 4520 MPT2SAS_DELL_PERC_H200_BRANDING); 4521 break; 4522 case MPT2SAS_DELL_6GBPS_SAS_SSDID: 4523 ioc_info(ioc, "%s\n", 4524 MPT2SAS_DELL_6GBPS_SAS_BRANDING); 4525 break; 4526 default: 4527 ioc_info(ioc, "Dell 6Gbps HBA: Subsystem ID: 0x%X\n", 4528 ioc->pdev->subsystem_device); 4529 break; 4530 } 4531 break; 4532 case MPI25_MFGPAGE_DEVID_SAS3008: 4533 switch (ioc->pdev->subsystem_device) { 4534 case MPT3SAS_DELL_12G_HBA_SSDID: 4535 ioc_info(ioc, "%s\n", 4536 MPT3SAS_DELL_12G_HBA_BRANDING); 4537 break; 4538 default: 4539 ioc_info(ioc, "Dell 12Gbps HBA: Subsystem ID: 0x%X\n", 4540 ioc->pdev->subsystem_device); 4541 break; 4542 } 4543 break; 4544 default: 4545 ioc_info(ioc, "Dell HBA: Subsystem ID: 0x%X\n", 4546 ioc->pdev->subsystem_device); 4547 break; 4548 } 4549 break; 4550 case PCI_VENDOR_ID_CISCO: 4551 switch (ioc->pdev->device) { 4552 case MPI25_MFGPAGE_DEVID_SAS3008: 4553 switch (ioc->pdev->subsystem_device) { 4554 case MPT3SAS_CISCO_12G_8E_HBA_SSDID: 4555 ioc_info(ioc, "%s\n", 4556 MPT3SAS_CISCO_12G_8E_HBA_BRANDING); 4557 break; 4558 case MPT3SAS_CISCO_12G_8I_HBA_SSDID: 4559 ioc_info(ioc, "%s\n", 4560 MPT3SAS_CISCO_12G_8I_HBA_BRANDING); 4561 break; 4562 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID: 4563 ioc_info(ioc, "%s\n", 4564 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING); 4565 break; 4566 default: 4567 ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n", 4568 ioc->pdev->subsystem_device); 4569 break; 4570 } 4571 break; 4572 case MPI25_MFGPAGE_DEVID_SAS3108_1: 4573 switch (ioc->pdev->subsystem_device) { 4574 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID: 4575 ioc_info(ioc, "%s\n", 4576 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING); 4577 break; 4578 case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID: 4579 ioc_info(ioc, "%s\n", 4580 MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING); 4581 break; 4582 default: 4583 ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n", 4584 ioc->pdev->subsystem_device); 4585 break; 4586 } 4587 break; 4588 default: 4589 ioc_info(ioc, "Cisco SAS HBA: Subsystem ID: 0x%X\n", 4590 ioc->pdev->subsystem_device); 4591 break; 4592 } 4593 break; 4594 case MPT2SAS_HP_3PAR_SSVID: 4595 switch (ioc->pdev->device) { 4596 case MPI2_MFGPAGE_DEVID_SAS2004: 4597 switch (ioc->pdev->subsystem_device) { 4598 case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID: 4599 ioc_info(ioc, "%s\n", 4600 MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING); 4601 break; 4602 default: 4603 ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n", 4604 ioc->pdev->subsystem_device); 4605 break; 4606 } 4607 break; 4608 case MPI2_MFGPAGE_DEVID_SAS2308_2: 4609 switch (ioc->pdev->subsystem_device) { 4610 case MPT2SAS_HP_2_4_INTERNAL_SSDID: 4611 ioc_info(ioc, "%s\n", 4612 MPT2SAS_HP_2_4_INTERNAL_BRANDING); 4613 break; 4614 case MPT2SAS_HP_2_4_EXTERNAL_SSDID: 4615 ioc_info(ioc, "%s\n", 4616 MPT2SAS_HP_2_4_EXTERNAL_BRANDING); 4617 break; 4618 case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID: 4619 ioc_info(ioc, "%s\n", 4620 MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING); 4621 break; 4622 case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID: 4623 ioc_info(ioc, "%s\n", 4624 MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING); 4625 break; 4626 default: 4627 ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n", 4628 ioc->pdev->subsystem_device); 4629 break; 4630 } 4631 break; 4632 default: 4633 ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n", 4634 ioc->pdev->subsystem_device); 4635 break; 4636 } 4637 break; 4638 default: 4639 break; 4640 } 4641 } 4642 4643 /** 4644 * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg 4645 * version from FW Image Header. 4646 * @ioc: per adapter object 4647 * 4648 * Return: 0 for success, non-zero for failure. 4649 */ 4650 static int 4651 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) 4652 { 4653 Mpi2FWImageHeader_t *fw_img_hdr; 4654 Mpi26ComponentImageHeader_t *cmp_img_hdr; 4655 Mpi25FWUploadRequest_t *mpi_request; 4656 Mpi2FWUploadReply_t mpi_reply; 4657 int r = 0, issue_diag_reset = 0; 4658 u32 package_version = 0; 4659 void *fwpkg_data = NULL; 4660 dma_addr_t fwpkg_data_dma; 4661 u16 smid, ioc_status; 4662 size_t data_length; 4663 4664 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 4665 4666 if (ioc->base_cmds.status & MPT3_CMD_PENDING) { 4667 ioc_err(ioc, "%s: internal command already in use\n", __func__); 4668 return -EAGAIN; 4669 } 4670 4671 data_length = sizeof(Mpi2FWImageHeader_t); 4672 fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length, 4673 &fwpkg_data_dma, GFP_KERNEL); 4674 if (!fwpkg_data) { 4675 ioc_err(ioc, 4676 "Memory allocation for fwpkg data failed at %s:%d/%s()!\n", 4677 __FILE__, __LINE__, __func__); 4678 return -ENOMEM; 4679 } 4680 4681 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 4682 if (!smid) { 4683 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 4684 r = -EAGAIN; 4685 goto out; 4686 } 4687 4688 ioc->base_cmds.status = MPT3_CMD_PENDING; 4689 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 4690 ioc->base_cmds.smid = smid; 4691 memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t)); 4692 mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD; 4693 mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH; 4694 mpi_request->ImageSize = cpu_to_le32(data_length); 4695 ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma, 4696 data_length); 4697 init_completion(&ioc->base_cmds.done); 4698 ioc->put_smid_default(ioc, smid); 4699 /* Wait for 15 seconds */ 4700 wait_for_completion_timeout(&ioc->base_cmds.done, 4701 FW_IMG_HDR_READ_TIMEOUT*HZ); 4702 ioc_info(ioc, "%s: complete\n", __func__); 4703 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 4704 ioc_err(ioc, "%s: timeout\n", __func__); 4705 _debug_dump_mf(mpi_request, 4706 sizeof(Mpi25FWUploadRequest_t)/4); 4707 issue_diag_reset = 1; 4708 } else { 4709 memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t)); 4710 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) { 4711 memcpy(&mpi_reply, ioc->base_cmds.reply, 4712 sizeof(Mpi2FWUploadReply_t)); 4713 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 4714 MPI2_IOCSTATUS_MASK; 4715 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) { 4716 fw_img_hdr = (Mpi2FWImageHeader_t *)fwpkg_data; 4717 if (le32_to_cpu(fw_img_hdr->Signature) == 4718 MPI26_IMAGE_HEADER_SIGNATURE0_MPI26) { 4719 cmp_img_hdr = 4720 (Mpi26ComponentImageHeader_t *) 4721 (fwpkg_data); 4722 package_version = 4723 le32_to_cpu( 4724 cmp_img_hdr->ApplicationSpecific); 4725 } else 4726 package_version = 4727 le32_to_cpu( 4728 fw_img_hdr->PackageVersion.Word); 4729 if (package_version) 4730 ioc_info(ioc, 4731 "FW Package Ver(%02d.%02d.%02d.%02d)\n", 4732 ((package_version) & 0xFF000000) >> 24, 4733 ((package_version) & 0x00FF0000) >> 16, 4734 ((package_version) & 0x0000FF00) >> 8, 4735 (package_version) & 0x000000FF); 4736 } else { 4737 _debug_dump_mf(&mpi_reply, 4738 sizeof(Mpi2FWUploadReply_t)/4); 4739 } 4740 } 4741 } 4742 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 4743 out: 4744 if (fwpkg_data) 4745 dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data, 4746 fwpkg_data_dma); 4747 if (issue_diag_reset) { 4748 if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED) 4749 return -EFAULT; 4750 if (mpt3sas_base_check_for_fault_and_issue_reset(ioc)) 4751 return -EFAULT; 4752 r = -EAGAIN; 4753 } 4754 return r; 4755 } 4756 4757 /** 4758 * _base_display_ioc_capabilities - Display IOC's capabilities. 4759 * @ioc: per adapter object 4760 */ 4761 static void 4762 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc) 4763 { 4764 int i = 0; 4765 char desc[16]; 4766 u32 iounit_pg1_flags; 4767 u32 bios_version; 4768 4769 bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion); 4770 strncpy(desc, ioc->manu_pg0.ChipName, 16); 4771 ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n", 4772 desc, 4773 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24, 4774 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16, 4775 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8, 4776 ioc->facts.FWVersion.Word & 0x000000FF, 4777 ioc->pdev->revision, 4778 (bios_version & 0xFF000000) >> 24, 4779 (bios_version & 0x00FF0000) >> 16, 4780 (bios_version & 0x0000FF00) >> 8, 4781 bios_version & 0x000000FF); 4782 4783 _base_display_OEMs_branding(ioc); 4784 4785 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) { 4786 pr_info("%sNVMe", i ? "," : ""); 4787 i++; 4788 } 4789 4790 ioc_info(ioc, "Protocol=("); 4791 4792 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) { 4793 pr_cont("Initiator"); 4794 i++; 4795 } 4796 4797 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) { 4798 pr_cont("%sTarget", i ? "," : ""); 4799 i++; 4800 } 4801 4802 i = 0; 4803 pr_cont("), Capabilities=("); 4804 4805 if (!ioc->hide_ir_msg) { 4806 if (ioc->facts.IOCCapabilities & 4807 MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) { 4808 pr_cont("Raid"); 4809 i++; 4810 } 4811 } 4812 4813 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) { 4814 pr_cont("%sTLR", i ? "," : ""); 4815 i++; 4816 } 4817 4818 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) { 4819 pr_cont("%sMulticast", i ? "," : ""); 4820 i++; 4821 } 4822 4823 if (ioc->facts.IOCCapabilities & 4824 MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) { 4825 pr_cont("%sBIDI Target", i ? "," : ""); 4826 i++; 4827 } 4828 4829 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) { 4830 pr_cont("%sEEDP", i ? "," : ""); 4831 i++; 4832 } 4833 4834 if (ioc->facts.IOCCapabilities & 4835 MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) { 4836 pr_cont("%sSnapshot Buffer", i ? "," : ""); 4837 i++; 4838 } 4839 4840 if (ioc->facts.IOCCapabilities & 4841 MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) { 4842 pr_cont("%sDiag Trace Buffer", i ? "," : ""); 4843 i++; 4844 } 4845 4846 if (ioc->facts.IOCCapabilities & 4847 MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) { 4848 pr_cont("%sDiag Extended Buffer", i ? "," : ""); 4849 i++; 4850 } 4851 4852 if (ioc->facts.IOCCapabilities & 4853 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) { 4854 pr_cont("%sTask Set Full", i ? "," : ""); 4855 i++; 4856 } 4857 4858 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags); 4859 if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) { 4860 pr_cont("%sNCQ", i ? "," : ""); 4861 i++; 4862 } 4863 4864 pr_cont(")\n"); 4865 } 4866 4867 /** 4868 * mpt3sas_base_update_missing_delay - change the missing delay timers 4869 * @ioc: per adapter object 4870 * @device_missing_delay: amount of time till device is reported missing 4871 * @io_missing_delay: interval IO is returned when there is a missing device 4872 * 4873 * Passed on the command line, this function will modify the device missing 4874 * delay, as well as the io missing delay. This should be called at driver 4875 * load time. 4876 */ 4877 void 4878 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc, 4879 u16 device_missing_delay, u8 io_missing_delay) 4880 { 4881 u16 dmd, dmd_new, dmd_orignal; 4882 u8 io_missing_delay_original; 4883 u16 sz; 4884 Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL; 4885 Mpi2ConfigReply_t mpi_reply; 4886 u8 num_phys = 0; 4887 u16 ioc_status; 4888 4889 mpt3sas_config_get_number_hba_phys(ioc, &num_phys); 4890 if (!num_phys) 4891 return; 4892 4893 sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys * 4894 sizeof(Mpi2SasIOUnit1PhyData_t)); 4895 sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL); 4896 if (!sas_iounit_pg1) { 4897 ioc_err(ioc, "failure at %s:%d/%s()!\n", 4898 __FILE__, __LINE__, __func__); 4899 goto out; 4900 } 4901 if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply, 4902 sas_iounit_pg1, sz))) { 4903 ioc_err(ioc, "failure at %s:%d/%s()!\n", 4904 __FILE__, __LINE__, __func__); 4905 goto out; 4906 } 4907 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 4908 MPI2_IOCSTATUS_MASK; 4909 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 4910 ioc_err(ioc, "failure at %s:%d/%s()!\n", 4911 __FILE__, __LINE__, __func__); 4912 goto out; 4913 } 4914 4915 /* device missing delay */ 4916 dmd = sas_iounit_pg1->ReportDeviceMissingDelay; 4917 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16) 4918 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16; 4919 else 4920 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK; 4921 dmd_orignal = dmd; 4922 if (device_missing_delay > 0x7F) { 4923 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 : 4924 device_missing_delay; 4925 dmd = dmd / 16; 4926 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16; 4927 } else 4928 dmd = device_missing_delay; 4929 sas_iounit_pg1->ReportDeviceMissingDelay = dmd; 4930 4931 /* io missing delay */ 4932 io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay; 4933 sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay; 4934 4935 if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1, 4936 sz)) { 4937 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16) 4938 dmd_new = (dmd & 4939 MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16; 4940 else 4941 dmd_new = 4942 dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK; 4943 ioc_info(ioc, "device_missing_delay: old(%d), new(%d)\n", 4944 dmd_orignal, dmd_new); 4945 ioc_info(ioc, "ioc_missing_delay: old(%d), new(%d)\n", 4946 io_missing_delay_original, 4947 io_missing_delay); 4948 ioc->device_missing_delay = dmd_new; 4949 ioc->io_missing_delay = io_missing_delay; 4950 } 4951 4952 out: 4953 kfree(sas_iounit_pg1); 4954 } 4955 4956 /** 4957 * _base_update_ioc_page1_inlinewith_perf_mode - Update IOC Page1 fields 4958 * according to performance mode. 4959 * @ioc : per adapter object 4960 * 4961 * Return: zero on success; otherwise return EAGAIN error code asking the 4962 * caller to retry. 4963 */ 4964 static int 4965 _base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc) 4966 { 4967 Mpi2IOCPage1_t ioc_pg1; 4968 Mpi2ConfigReply_t mpi_reply; 4969 int rc; 4970 4971 rc = mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy); 4972 if (rc) 4973 return rc; 4974 memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t)); 4975 4976 switch (perf_mode) { 4977 case MPT_PERF_MODE_DEFAULT: 4978 case MPT_PERF_MODE_BALANCED: 4979 if (ioc->high_iops_queues) { 4980 ioc_info(ioc, 4981 "Enable interrupt coalescing only for first\t" 4982 "%d reply queues\n", 4983 MPT3SAS_HIGH_IOPS_REPLY_QUEUES); 4984 /* 4985 * If 31st bit is zero then interrupt coalescing is 4986 * enabled for all reply descriptor post queues. 4987 * If 31st bit is set to one then user can 4988 * enable/disable interrupt coalescing on per reply 4989 * descriptor post queue group(8) basis. So to enable 4990 * interrupt coalescing only on first reply descriptor 4991 * post queue group 31st bit and zero th bit is enabled. 4992 */ 4993 ioc_pg1.ProductSpecific = cpu_to_le32(0x80000000 | 4994 ((1 << MPT3SAS_HIGH_IOPS_REPLY_QUEUES/8) - 1)); 4995 rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); 4996 if (rc) 4997 return rc; 4998 ioc_info(ioc, "performance mode: balanced\n"); 4999 return 0; 5000 } 5001 fallthrough; 5002 case MPT_PERF_MODE_LATENCY: 5003 /* 5004 * Enable interrupt coalescing on all reply queues 5005 * with timeout value 0xA 5006 */ 5007 ioc_pg1.CoalescingTimeout = cpu_to_le32(0xa); 5008 ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING); 5009 ioc_pg1.ProductSpecific = 0; 5010 rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); 5011 if (rc) 5012 return rc; 5013 ioc_info(ioc, "performance mode: latency\n"); 5014 break; 5015 case MPT_PERF_MODE_IOPS: 5016 /* 5017 * Enable interrupt coalescing on all reply queues. 5018 */ 5019 ioc_info(ioc, 5020 "performance mode: iops with coalescing timeout: 0x%x\n", 5021 le32_to_cpu(ioc_pg1.CoalescingTimeout)); 5022 ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING); 5023 ioc_pg1.ProductSpecific = 0; 5024 rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); 5025 if (rc) 5026 return rc; 5027 break; 5028 } 5029 return 0; 5030 } 5031 5032 /** 5033 * _base_get_event_diag_triggers - get event diag trigger values from 5034 * persistent pages 5035 * @ioc : per adapter object 5036 * 5037 * Return: nothing. 5038 */ 5039 static int 5040 _base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5041 { 5042 Mpi26DriverTriggerPage2_t trigger_pg2; 5043 struct SL_WH_EVENT_TRIGGER_T *event_tg; 5044 MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY *mpi_event_tg; 5045 Mpi2ConfigReply_t mpi_reply; 5046 int r = 0, i = 0; 5047 u16 count = 0; 5048 u16 ioc_status; 5049 5050 r = mpt3sas_config_get_driver_trigger_pg2(ioc, &mpi_reply, 5051 &trigger_pg2); 5052 if (r) 5053 return r; 5054 5055 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5056 MPI2_IOCSTATUS_MASK; 5057 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5058 dinitprintk(ioc, 5059 ioc_err(ioc, 5060 "%s: Failed to get trigger pg2, ioc_status(0x%04x)\n", 5061 __func__, ioc_status)); 5062 return 0; 5063 } 5064 5065 if (le16_to_cpu(trigger_pg2.NumMPIEventTrigger)) { 5066 count = le16_to_cpu(trigger_pg2.NumMPIEventTrigger); 5067 count = min_t(u16, NUM_VALID_ENTRIES, count); 5068 ioc->diag_trigger_event.ValidEntries = count; 5069 5070 event_tg = &ioc->diag_trigger_event.EventTriggerEntry[0]; 5071 mpi_event_tg = &trigger_pg2.MPIEventTriggers[0]; 5072 for (i = 0; i < count; i++) { 5073 event_tg->EventValue = le16_to_cpu( 5074 mpi_event_tg->MPIEventCode); 5075 event_tg->LogEntryQualifier = le16_to_cpu( 5076 mpi_event_tg->MPIEventCodeSpecific); 5077 event_tg++; 5078 mpi_event_tg++; 5079 } 5080 } 5081 return 0; 5082 } 5083 5084 /** 5085 * _base_get_scsi_diag_triggers - get scsi diag trigger values from 5086 * persistent pages 5087 * @ioc : per adapter object 5088 * 5089 * Return: 0 on success; otherwise return failure status. 5090 */ 5091 static int 5092 _base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5093 { 5094 Mpi26DriverTriggerPage3_t trigger_pg3; 5095 struct SL_WH_SCSI_TRIGGER_T *scsi_tg; 5096 MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY *mpi_scsi_tg; 5097 Mpi2ConfigReply_t mpi_reply; 5098 int r = 0, i = 0; 5099 u16 count = 0; 5100 u16 ioc_status; 5101 5102 r = mpt3sas_config_get_driver_trigger_pg3(ioc, &mpi_reply, 5103 &trigger_pg3); 5104 if (r) 5105 return r; 5106 5107 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5108 MPI2_IOCSTATUS_MASK; 5109 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5110 dinitprintk(ioc, 5111 ioc_err(ioc, 5112 "%s: Failed to get trigger pg3, ioc_status(0x%04x)\n", 5113 __func__, ioc_status)); 5114 return 0; 5115 } 5116 5117 if (le16_to_cpu(trigger_pg3.NumSCSISenseTrigger)) { 5118 count = le16_to_cpu(trigger_pg3.NumSCSISenseTrigger); 5119 count = min_t(u16, NUM_VALID_ENTRIES, count); 5120 ioc->diag_trigger_scsi.ValidEntries = count; 5121 5122 scsi_tg = &ioc->diag_trigger_scsi.SCSITriggerEntry[0]; 5123 mpi_scsi_tg = &trigger_pg3.SCSISenseTriggers[0]; 5124 for (i = 0; i < count; i++) { 5125 scsi_tg->ASCQ = mpi_scsi_tg->ASCQ; 5126 scsi_tg->ASC = mpi_scsi_tg->ASC; 5127 scsi_tg->SenseKey = mpi_scsi_tg->SenseKey; 5128 5129 scsi_tg++; 5130 mpi_scsi_tg++; 5131 } 5132 } 5133 return 0; 5134 } 5135 5136 /** 5137 * _base_get_mpi_diag_triggers - get mpi diag trigger values from 5138 * persistent pages 5139 * @ioc : per adapter object 5140 * 5141 * Return: 0 on success; otherwise return failure status. 5142 */ 5143 static int 5144 _base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5145 { 5146 Mpi26DriverTriggerPage4_t trigger_pg4; 5147 struct SL_WH_MPI_TRIGGER_T *status_tg; 5148 MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY *mpi_status_tg; 5149 Mpi2ConfigReply_t mpi_reply; 5150 int r = 0, i = 0; 5151 u16 count = 0; 5152 u16 ioc_status; 5153 5154 r = mpt3sas_config_get_driver_trigger_pg4(ioc, &mpi_reply, 5155 &trigger_pg4); 5156 if (r) 5157 return r; 5158 5159 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5160 MPI2_IOCSTATUS_MASK; 5161 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5162 dinitprintk(ioc, 5163 ioc_err(ioc, 5164 "%s: Failed to get trigger pg4, ioc_status(0x%04x)\n", 5165 __func__, ioc_status)); 5166 return 0; 5167 } 5168 5169 if (le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger)) { 5170 count = le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger); 5171 count = min_t(u16, NUM_VALID_ENTRIES, count); 5172 ioc->diag_trigger_mpi.ValidEntries = count; 5173 5174 status_tg = &ioc->diag_trigger_mpi.MPITriggerEntry[0]; 5175 mpi_status_tg = &trigger_pg4.IOCStatusLoginfoTriggers[0]; 5176 5177 for (i = 0; i < count; i++) { 5178 status_tg->IOCStatus = le16_to_cpu( 5179 mpi_status_tg->IOCStatus); 5180 status_tg->IocLogInfo = le32_to_cpu( 5181 mpi_status_tg->LogInfo); 5182 5183 status_tg++; 5184 mpi_status_tg++; 5185 } 5186 } 5187 return 0; 5188 } 5189 5190 /** 5191 * _base_get_master_diag_triggers - get master diag trigger values from 5192 * persistent pages 5193 * @ioc : per adapter object 5194 * 5195 * Return: nothing. 5196 */ 5197 static int 5198 _base_get_master_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5199 { 5200 Mpi26DriverTriggerPage1_t trigger_pg1; 5201 Mpi2ConfigReply_t mpi_reply; 5202 int r; 5203 u16 ioc_status; 5204 5205 r = mpt3sas_config_get_driver_trigger_pg1(ioc, &mpi_reply, 5206 &trigger_pg1); 5207 if (r) 5208 return r; 5209 5210 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5211 MPI2_IOCSTATUS_MASK; 5212 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5213 dinitprintk(ioc, 5214 ioc_err(ioc, 5215 "%s: Failed to get trigger pg1, ioc_status(0x%04x)\n", 5216 __func__, ioc_status)); 5217 return 0; 5218 } 5219 5220 if (le16_to_cpu(trigger_pg1.NumMasterTrigger)) 5221 ioc->diag_trigger_master.MasterData |= 5222 le32_to_cpu( 5223 trigger_pg1.MasterTriggers[0].MasterTriggerFlags); 5224 return 0; 5225 } 5226 5227 /** 5228 * _base_check_for_trigger_pages_support - checks whether HBA FW supports 5229 * driver trigger pages or not 5230 * @ioc : per adapter object 5231 * @trigger_flags : address where trigger page0's TriggerFlags value is copied 5232 * 5233 * Return: trigger flags mask if HBA FW supports driver trigger pages; 5234 * otherwise returns %-EFAULT if driver trigger pages are not supported by FW or 5235 * return EAGAIN if diag reset occurred due to FW fault and asking the 5236 * caller to retry the command. 5237 * 5238 */ 5239 static int 5240 _base_check_for_trigger_pages_support(struct MPT3SAS_ADAPTER *ioc, u32 *trigger_flags) 5241 { 5242 Mpi26DriverTriggerPage0_t trigger_pg0; 5243 int r = 0; 5244 Mpi2ConfigReply_t mpi_reply; 5245 u16 ioc_status; 5246 5247 r = mpt3sas_config_get_driver_trigger_pg0(ioc, &mpi_reply, 5248 &trigger_pg0); 5249 if (r) 5250 return r; 5251 5252 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5253 MPI2_IOCSTATUS_MASK; 5254 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) 5255 return -EFAULT; 5256 5257 *trigger_flags = le16_to_cpu(trigger_pg0.TriggerFlags); 5258 return 0; 5259 } 5260 5261 /** 5262 * _base_get_diag_triggers - Retrieve diag trigger values from 5263 * persistent pages. 5264 * @ioc : per adapter object 5265 * 5266 * Return: zero on success; otherwise return EAGAIN error codes 5267 * asking the caller to retry. 5268 */ 5269 static int 5270 _base_get_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5271 { 5272 int trigger_flags; 5273 int r; 5274 5275 /* 5276 * Default setting of master trigger. 5277 */ 5278 ioc->diag_trigger_master.MasterData = 5279 (MASTER_TRIGGER_FW_FAULT + MASTER_TRIGGER_ADAPTER_RESET); 5280 5281 r = _base_check_for_trigger_pages_support(ioc, &trigger_flags); 5282 if (r) { 5283 if (r == -EAGAIN) 5284 return r; 5285 /* 5286 * Don't go for error handling when FW doesn't support 5287 * driver trigger pages. 5288 */ 5289 return 0; 5290 } 5291 5292 ioc->supports_trigger_pages = 1; 5293 5294 /* 5295 * Retrieve master diag trigger values from driver trigger pg1 5296 * if master trigger bit enabled in TriggerFlags. 5297 */ 5298 if ((u16)trigger_flags & 5299 MPI26_DRIVER_TRIGGER0_FLAG_MASTER_TRIGGER_VALID) { 5300 r = _base_get_master_diag_triggers(ioc); 5301 if (r) 5302 return r; 5303 } 5304 5305 /* 5306 * Retrieve event diag trigger values from driver trigger pg2 5307 * if event trigger bit enabled in TriggerFlags. 5308 */ 5309 if ((u16)trigger_flags & 5310 MPI26_DRIVER_TRIGGER0_FLAG_MPI_EVENT_TRIGGER_VALID) { 5311 r = _base_get_event_diag_triggers(ioc); 5312 if (r) 5313 return r; 5314 } 5315 5316 /* 5317 * Retrieve scsi diag trigger values from driver trigger pg3 5318 * if scsi trigger bit enabled in TriggerFlags. 5319 */ 5320 if ((u16)trigger_flags & 5321 MPI26_DRIVER_TRIGGER0_FLAG_SCSI_SENSE_TRIGGER_VALID) { 5322 r = _base_get_scsi_diag_triggers(ioc); 5323 if (r) 5324 return r; 5325 } 5326 /* 5327 * Retrieve mpi error diag trigger values from driver trigger pg4 5328 * if loginfo trigger bit enabled in TriggerFlags. 5329 */ 5330 if ((u16)trigger_flags & 5331 MPI26_DRIVER_TRIGGER0_FLAG_LOGINFO_TRIGGER_VALID) { 5332 r = _base_get_mpi_diag_triggers(ioc); 5333 if (r) 5334 return r; 5335 } 5336 return 0; 5337 } 5338 5339 /** 5340 * _base_update_diag_trigger_pages - Update the driver trigger pages after 5341 * online FW update, in case updated FW supports driver 5342 * trigger pages. 5343 * @ioc : per adapter object 5344 * 5345 * Return: nothing. 5346 */ 5347 static void 5348 _base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc) 5349 { 5350 5351 if (ioc->diag_trigger_master.MasterData) 5352 mpt3sas_config_update_driver_trigger_pg1(ioc, 5353 &ioc->diag_trigger_master, 1); 5354 5355 if (ioc->diag_trigger_event.ValidEntries) 5356 mpt3sas_config_update_driver_trigger_pg2(ioc, 5357 &ioc->diag_trigger_event, 1); 5358 5359 if (ioc->diag_trigger_scsi.ValidEntries) 5360 mpt3sas_config_update_driver_trigger_pg3(ioc, 5361 &ioc->diag_trigger_scsi, 1); 5362 5363 if (ioc->diag_trigger_mpi.ValidEntries) 5364 mpt3sas_config_update_driver_trigger_pg4(ioc, 5365 &ioc->diag_trigger_mpi, 1); 5366 } 5367 5368 /** 5369 * _base_assign_fw_reported_qd - Get FW reported QD for SAS/SATA devices. 5370 * - On failure set default QD values. 5371 * @ioc : per adapter object 5372 * 5373 * Returns 0 for success, non-zero for failure. 5374 * 5375 */ 5376 static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc) 5377 { 5378 Mpi2ConfigReply_t mpi_reply; 5379 Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL; 5380 Mpi26PCIeIOUnitPage1_t pcie_iounit_pg1; 5381 int sz; 5382 int rc = 0; 5383 5384 ioc->max_wideport_qd = MPT3SAS_SAS_QUEUE_DEPTH; 5385 ioc->max_narrowport_qd = MPT3SAS_SAS_QUEUE_DEPTH; 5386 ioc->max_sata_qd = MPT3SAS_SATA_QUEUE_DEPTH; 5387 ioc->max_nvme_qd = MPT3SAS_NVME_QUEUE_DEPTH; 5388 if (!ioc->is_gen35_ioc) 5389 goto out; 5390 /* sas iounit page 1 */ 5391 sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData); 5392 sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL); 5393 if (!sas_iounit_pg1) { 5394 pr_err("%s: failure at %s:%d/%s()!\n", 5395 ioc->name, __FILE__, __LINE__, __func__); 5396 return rc; 5397 } 5398 rc = mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply, 5399 sas_iounit_pg1, sz); 5400 if (rc) { 5401 pr_err("%s: failure at %s:%d/%s()!\n", 5402 ioc->name, __FILE__, __LINE__, __func__); 5403 goto out; 5404 } 5405 ioc->max_wideport_qd = 5406 (le16_to_cpu(sas_iounit_pg1->SASWideMaxQueueDepth)) ? 5407 le16_to_cpu(sas_iounit_pg1->SASWideMaxQueueDepth) : 5408 MPT3SAS_SAS_QUEUE_DEPTH; 5409 ioc->max_narrowport_qd = 5410 (le16_to_cpu(sas_iounit_pg1->SASNarrowMaxQueueDepth)) ? 5411 le16_to_cpu(sas_iounit_pg1->SASNarrowMaxQueueDepth) : 5412 MPT3SAS_SAS_QUEUE_DEPTH; 5413 ioc->max_sata_qd = (sas_iounit_pg1->SATAMaxQDepth) ? 5414 sas_iounit_pg1->SATAMaxQDepth : MPT3SAS_SATA_QUEUE_DEPTH; 5415 /* pcie iounit page 1 */ 5416 rc = mpt3sas_config_get_pcie_iounit_pg1(ioc, &mpi_reply, 5417 &pcie_iounit_pg1, sizeof(Mpi26PCIeIOUnitPage1_t)); 5418 if (rc) { 5419 pr_err("%s: failure at %s:%d/%s()!\n", 5420 ioc->name, __FILE__, __LINE__, __func__); 5421 goto out; 5422 } 5423 ioc->max_nvme_qd = (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) ? 5424 (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) : 5425 MPT3SAS_NVME_QUEUE_DEPTH; 5426 out: 5427 dinitprintk(ioc, pr_err( 5428 "MaxWidePortQD: 0x%x MaxNarrowPortQD: 0x%x MaxSataQD: 0x%x MaxNvmeQD: 0x%x\n", 5429 ioc->max_wideport_qd, ioc->max_narrowport_qd, 5430 ioc->max_sata_qd, ioc->max_nvme_qd)); 5431 kfree(sas_iounit_pg1); 5432 return rc; 5433 } 5434 5435 /** 5436 * _base_static_config_pages - static start of day config pages 5437 * @ioc: per adapter object 5438 */ 5439 static int 5440 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) 5441 { 5442 Mpi2ConfigReply_t mpi_reply; 5443 u32 iounit_pg1_flags; 5444 int tg_flags = 0; 5445 int rc; 5446 ioc->nvme_abort_timeout = 30; 5447 5448 rc = mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, 5449 &ioc->manu_pg0); 5450 if (rc) 5451 return rc; 5452 if (ioc->ir_firmware) { 5453 rc = mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply, 5454 &ioc->manu_pg10); 5455 if (rc) 5456 return rc; 5457 } 5458 /* 5459 * Ensure correct T10 PI operation if vendor left EEDPTagMode 5460 * flag unset in NVDATA. 5461 */ 5462 rc = mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, 5463 &ioc->manu_pg11); 5464 if (rc) 5465 return rc; 5466 if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) { 5467 pr_err("%s: overriding NVDATA EEDPTagMode setting\n", 5468 ioc->name); 5469 ioc->manu_pg11.EEDPTagMode &= ~0x3; 5470 ioc->manu_pg11.EEDPTagMode |= 0x1; 5471 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply, 5472 &ioc->manu_pg11); 5473 } 5474 if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK) 5475 ioc->tm_custom_handling = 1; 5476 else { 5477 ioc->tm_custom_handling = 0; 5478 if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT) 5479 ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT; 5480 else if (ioc->manu_pg11.NVMeAbortTO > 5481 NVME_TASK_ABORT_MAX_TIMEOUT) 5482 ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT; 5483 else 5484 ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO; 5485 } 5486 ioc->time_sync_interval = 5487 ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_MASK; 5488 if (ioc->time_sync_interval) { 5489 if (ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_UNIT_MASK) 5490 ioc->time_sync_interval = 5491 ioc->time_sync_interval * SECONDS_PER_HOUR; 5492 else 5493 ioc->time_sync_interval = 5494 ioc->time_sync_interval * SECONDS_PER_MIN; 5495 dinitprintk(ioc, ioc_info(ioc, 5496 "Driver-FW TimeSync interval is %d seconds. ManuPg11 TimeSync Unit is in %s\n", 5497 ioc->time_sync_interval, (ioc->manu_pg11.TimeSyncInterval & 5498 MPT3SAS_TIMESYNC_UNIT_MASK) ? "Hour" : "Minute")); 5499 } else { 5500 if (ioc->is_gen35_ioc) 5501 ioc_warn(ioc, 5502 "TimeSync Interval in Manuf page-11 is not enabled. Periodic Time-Sync will be disabled\n"); 5503 } 5504 rc = _base_assign_fw_reported_qd(ioc); 5505 if (rc) 5506 return rc; 5507 rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2); 5508 if (rc) 5509 return rc; 5510 rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3); 5511 if (rc) 5512 return rc; 5513 rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8); 5514 if (rc) 5515 return rc; 5516 rc = mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0); 5517 if (rc) 5518 return rc; 5519 rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); 5520 if (rc) 5521 return rc; 5522 rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8); 5523 if (rc) 5524 return rc; 5525 _base_display_ioc_capabilities(ioc); 5526 5527 /* 5528 * Enable task_set_full handling in iounit_pg1 when the 5529 * facts capabilities indicate that its supported. 5530 */ 5531 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags); 5532 if ((ioc->facts.IOCCapabilities & 5533 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING)) 5534 iounit_pg1_flags &= 5535 ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING; 5536 else 5537 iounit_pg1_flags |= 5538 MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING; 5539 ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags); 5540 rc = mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); 5541 if (rc) 5542 return rc; 5543 5544 if (ioc->iounit_pg8.NumSensors) 5545 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors; 5546 if (ioc->is_aero_ioc) { 5547 rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc); 5548 if (rc) 5549 return rc; 5550 } 5551 if (ioc->is_gen35_ioc) { 5552 if (ioc->is_driver_loading) { 5553 rc = _base_get_diag_triggers(ioc); 5554 if (rc) 5555 return rc; 5556 } else { 5557 /* 5558 * In case of online HBA FW update operation, 5559 * check whether updated FW supports the driver trigger 5560 * pages or not. 5561 * - If previous FW has not supported driver trigger 5562 * pages and newer FW supports them then update these 5563 * pages with current diag trigger values. 5564 * - If previous FW has supported driver trigger pages 5565 * and new FW doesn't support them then disable 5566 * support_trigger_pages flag. 5567 */ 5568 _base_check_for_trigger_pages_support(ioc, &tg_flags); 5569 if (!ioc->supports_trigger_pages && tg_flags != -EFAULT) 5570 _base_update_diag_trigger_pages(ioc); 5571 else if (ioc->supports_trigger_pages && 5572 tg_flags == -EFAULT) 5573 ioc->supports_trigger_pages = 0; 5574 } 5575 } 5576 return 0; 5577 } 5578 5579 /** 5580 * mpt3sas_free_enclosure_list - release memory 5581 * @ioc: per adapter object 5582 * 5583 * Free memory allocated during enclosure add. 5584 */ 5585 void 5586 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc) 5587 { 5588 struct _enclosure_node *enclosure_dev, *enclosure_dev_next; 5589 5590 /* Free enclosure list */ 5591 list_for_each_entry_safe(enclosure_dev, 5592 enclosure_dev_next, &ioc->enclosure_list, list) { 5593 list_del(&enclosure_dev->list); 5594 kfree(enclosure_dev); 5595 } 5596 } 5597 5598 /** 5599 * _base_release_memory_pools - release memory 5600 * @ioc: per adapter object 5601 * 5602 * Free memory allocated from _base_allocate_memory_pools. 5603 */ 5604 static void 5605 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) 5606 { 5607 int i = 0; 5608 int j = 0; 5609 int dma_alloc_count = 0; 5610 struct chain_tracker *ct; 5611 int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1; 5612 5613 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 5614 5615 if (ioc->request) { 5616 dma_free_coherent(&ioc->pdev->dev, ioc->request_dma_sz, 5617 ioc->request, ioc->request_dma); 5618 dexitprintk(ioc, 5619 ioc_info(ioc, "request_pool(0x%p): free\n", 5620 ioc->request)); 5621 ioc->request = NULL; 5622 } 5623 5624 if (ioc->sense) { 5625 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma); 5626 dma_pool_destroy(ioc->sense_dma_pool); 5627 dexitprintk(ioc, 5628 ioc_info(ioc, "sense_pool(0x%p): free\n", 5629 ioc->sense)); 5630 ioc->sense = NULL; 5631 } 5632 5633 if (ioc->reply) { 5634 dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma); 5635 dma_pool_destroy(ioc->reply_dma_pool); 5636 dexitprintk(ioc, 5637 ioc_info(ioc, "reply_pool(0x%p): free\n", 5638 ioc->reply)); 5639 ioc->reply = NULL; 5640 } 5641 5642 if (ioc->reply_free) { 5643 dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free, 5644 ioc->reply_free_dma); 5645 dma_pool_destroy(ioc->reply_free_dma_pool); 5646 dexitprintk(ioc, 5647 ioc_info(ioc, "reply_free_pool(0x%p): free\n", 5648 ioc->reply_free)); 5649 ioc->reply_free = NULL; 5650 } 5651 5652 if (ioc->reply_post) { 5653 dma_alloc_count = DIV_ROUND_UP(count, 5654 RDPQ_MAX_INDEX_IN_ONE_CHUNK); 5655 for (i = 0; i < count; i++) { 5656 if (i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0 5657 && dma_alloc_count) { 5658 if (ioc->reply_post[i].reply_post_free) { 5659 dma_pool_free( 5660 ioc->reply_post_free_dma_pool, 5661 ioc->reply_post[i].reply_post_free, 5662 ioc->reply_post[i].reply_post_free_dma); 5663 dexitprintk(ioc, ioc_info(ioc, 5664 "reply_post_free_pool(0x%p): free\n", 5665 ioc->reply_post[i].reply_post_free)); 5666 ioc->reply_post[i].reply_post_free = 5667 NULL; 5668 } 5669 --dma_alloc_count; 5670 } 5671 } 5672 dma_pool_destroy(ioc->reply_post_free_dma_pool); 5673 if (ioc->reply_post_free_array && 5674 ioc->rdpq_array_enable) { 5675 dma_pool_free(ioc->reply_post_free_array_dma_pool, 5676 ioc->reply_post_free_array, 5677 ioc->reply_post_free_array_dma); 5678 ioc->reply_post_free_array = NULL; 5679 } 5680 dma_pool_destroy(ioc->reply_post_free_array_dma_pool); 5681 kfree(ioc->reply_post); 5682 } 5683 5684 if (ioc->pcie_sgl_dma_pool) { 5685 for (i = 0; i < ioc->scsiio_depth; i++) { 5686 dma_pool_free(ioc->pcie_sgl_dma_pool, 5687 ioc->pcie_sg_lookup[i].pcie_sgl, 5688 ioc->pcie_sg_lookup[i].pcie_sgl_dma); 5689 ioc->pcie_sg_lookup[i].pcie_sgl = NULL; 5690 } 5691 dma_pool_destroy(ioc->pcie_sgl_dma_pool); 5692 } 5693 if (ioc->config_page) { 5694 dexitprintk(ioc, 5695 ioc_info(ioc, "config_page(0x%p): free\n", 5696 ioc->config_page)); 5697 dma_free_coherent(&ioc->pdev->dev, ioc->config_page_sz, 5698 ioc->config_page, ioc->config_page_dma); 5699 } 5700 5701 kfree(ioc->hpr_lookup); 5702 ioc->hpr_lookup = NULL; 5703 kfree(ioc->internal_lookup); 5704 ioc->internal_lookup = NULL; 5705 if (ioc->chain_lookup) { 5706 for (i = 0; i < ioc->scsiio_depth; i++) { 5707 for (j = ioc->chains_per_prp_buffer; 5708 j < ioc->chains_needed_per_io; j++) { 5709 ct = &ioc->chain_lookup[i].chains_per_smid[j]; 5710 if (ct && ct->chain_buffer) 5711 dma_pool_free(ioc->chain_dma_pool, 5712 ct->chain_buffer, 5713 ct->chain_buffer_dma); 5714 } 5715 kfree(ioc->chain_lookup[i].chains_per_smid); 5716 } 5717 dma_pool_destroy(ioc->chain_dma_pool); 5718 kfree(ioc->chain_lookup); 5719 ioc->chain_lookup = NULL; 5720 } 5721 5722 kfree(ioc->io_queue_num); 5723 ioc->io_queue_num = NULL; 5724 } 5725 5726 /** 5727 * mpt3sas_check_same_4gb_region - checks whether all reply queues in a set are 5728 * having same upper 32bits in their base memory address. 5729 * @reply_pool_start_address: Base address of a reply queue set 5730 * @pool_sz: Size of single Reply Descriptor Post Queues pool size 5731 * 5732 * Return: 1 if reply queues in a set have a same upper 32bits in their base 5733 * memory address, else 0. 5734 */ 5735 5736 static int 5737 mpt3sas_check_same_4gb_region(long reply_pool_start_address, u32 pool_sz) 5738 { 5739 long reply_pool_end_address; 5740 5741 reply_pool_end_address = reply_pool_start_address + pool_sz; 5742 5743 if (upper_32_bits(reply_pool_start_address) == 5744 upper_32_bits(reply_pool_end_address)) 5745 return 1; 5746 else 5747 return 0; 5748 } 5749 5750 /** 5751 * _base_reduce_hba_queue_depth- Retry with reduced queue depth 5752 * @ioc: Adapter object 5753 * 5754 * Return: 0 for success, non-zero for failure. 5755 **/ 5756 static inline int 5757 _base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc) 5758 { 5759 int reduce_sz = 64; 5760 5761 if ((ioc->hba_queue_depth - reduce_sz) > 5762 (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) { 5763 ioc->hba_queue_depth -= reduce_sz; 5764 return 0; 5765 } else 5766 return -ENOMEM; 5767 } 5768 5769 /** 5770 * _base_allocate_pcie_sgl_pool - Allocating DMA'able memory 5771 * for pcie sgl pools. 5772 * @ioc: Adapter object 5773 * @sz: DMA Pool size 5774 * 5775 * Return: 0 for success, non-zero for failure. 5776 */ 5777 5778 static int 5779 _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5780 { 5781 int i = 0, j = 0; 5782 struct chain_tracker *ct; 5783 5784 ioc->pcie_sgl_dma_pool = 5785 dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, 5786 ioc->page_size, 0); 5787 if (!ioc->pcie_sgl_dma_pool) { 5788 ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n"); 5789 return -ENOMEM; 5790 } 5791 5792 ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz; 5793 ioc->chains_per_prp_buffer = 5794 min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io); 5795 for (i = 0; i < ioc->scsiio_depth; i++) { 5796 ioc->pcie_sg_lookup[i].pcie_sgl = 5797 dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL, 5798 &ioc->pcie_sg_lookup[i].pcie_sgl_dma); 5799 if (!ioc->pcie_sg_lookup[i].pcie_sgl) { 5800 ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n"); 5801 return -EAGAIN; 5802 } 5803 5804 if (!mpt3sas_check_same_4gb_region( 5805 (long)ioc->pcie_sg_lookup[i].pcie_sgl, sz)) { 5806 ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n", 5807 ioc->pcie_sg_lookup[i].pcie_sgl, 5808 (unsigned long long) 5809 ioc->pcie_sg_lookup[i].pcie_sgl_dma); 5810 ioc->use_32bit_dma = true; 5811 return -EAGAIN; 5812 } 5813 5814 for (j = 0; j < ioc->chains_per_prp_buffer; j++) { 5815 ct = &ioc->chain_lookup[i].chains_per_smid[j]; 5816 ct->chain_buffer = 5817 ioc->pcie_sg_lookup[i].pcie_sgl + 5818 (j * ioc->chain_segment_sz); 5819 ct->chain_buffer_dma = 5820 ioc->pcie_sg_lookup[i].pcie_sgl_dma + 5821 (j * ioc->chain_segment_sz); 5822 } 5823 } 5824 dinitprintk(ioc, ioc_info(ioc, 5825 "PCIe sgl pool depth(%d), element_size(%d), pool_size(%d kB)\n", 5826 ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024)); 5827 dinitprintk(ioc, ioc_info(ioc, 5828 "Number of chains can fit in a PRP page(%d)\n", 5829 ioc->chains_per_prp_buffer)); 5830 return 0; 5831 } 5832 5833 /** 5834 * _base_allocate_chain_dma_pool - Allocating DMA'able memory 5835 * for chain dma pool. 5836 * @ioc: Adapter object 5837 * @sz: DMA Pool size 5838 * 5839 * Return: 0 for success, non-zero for failure. 5840 */ 5841 static int 5842 _base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5843 { 5844 int i = 0, j = 0; 5845 struct chain_tracker *ctr; 5846 5847 ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev, 5848 ioc->chain_segment_sz, 16, 0); 5849 if (!ioc->chain_dma_pool) 5850 return -ENOMEM; 5851 5852 for (i = 0; i < ioc->scsiio_depth; i++) { 5853 for (j = ioc->chains_per_prp_buffer; 5854 j < ioc->chains_needed_per_io; j++) { 5855 ctr = &ioc->chain_lookup[i].chains_per_smid[j]; 5856 ctr->chain_buffer = dma_pool_alloc(ioc->chain_dma_pool, 5857 GFP_KERNEL, &ctr->chain_buffer_dma); 5858 if (!ctr->chain_buffer) 5859 return -EAGAIN; 5860 if (!mpt3sas_check_same_4gb_region((long) 5861 ctr->chain_buffer, ioc->chain_segment_sz)) { 5862 ioc_err(ioc, 5863 "Chain buffers are not in same 4G !!! Chain buff (0x%p) dma = (0x%llx)\n", 5864 ctr->chain_buffer, 5865 (unsigned long long)ctr->chain_buffer_dma); 5866 ioc->use_32bit_dma = true; 5867 return -EAGAIN; 5868 } 5869 } 5870 } 5871 dinitprintk(ioc, ioc_info(ioc, 5872 "chain_lookup depth (%d), frame_size(%d), pool_size(%d kB)\n", 5873 ioc->scsiio_depth, ioc->chain_segment_sz, ((ioc->scsiio_depth * 5874 (ioc->chains_needed_per_io - ioc->chains_per_prp_buffer) * 5875 ioc->chain_segment_sz))/1024)); 5876 return 0; 5877 } 5878 5879 /** 5880 * _base_allocate_sense_dma_pool - Allocating DMA'able memory 5881 * for sense dma pool. 5882 * @ioc: Adapter object 5883 * @sz: DMA Pool size 5884 * Return: 0 for success, non-zero for failure. 5885 */ 5886 static int 5887 _base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5888 { 5889 ioc->sense_dma_pool = 5890 dma_pool_create("sense pool", &ioc->pdev->dev, sz, 4, 0); 5891 if (!ioc->sense_dma_pool) 5892 return -ENOMEM; 5893 ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, 5894 GFP_KERNEL, &ioc->sense_dma); 5895 if (!ioc->sense) 5896 return -EAGAIN; 5897 if (!mpt3sas_check_same_4gb_region((long)ioc->sense, sz)) { 5898 dinitprintk(ioc, pr_err( 5899 "Bad Sense Pool! sense (0x%p) sense_dma = (0x%llx)\n", 5900 ioc->sense, (unsigned long long) ioc->sense_dma)); 5901 ioc->use_32bit_dma = true; 5902 return -EAGAIN; 5903 } 5904 ioc_info(ioc, 5905 "sense pool(0x%p) - dma(0x%llx): depth(%d), element_size(%d), pool_size (%d kB)\n", 5906 ioc->sense, (unsigned long long)ioc->sense_dma, 5907 ioc->scsiio_depth, SCSI_SENSE_BUFFERSIZE, sz/1024); 5908 return 0; 5909 } 5910 5911 /** 5912 * _base_allocate_reply_pool - Allocating DMA'able memory 5913 * for reply pool. 5914 * @ioc: Adapter object 5915 * @sz: DMA Pool size 5916 * Return: 0 for success, non-zero for failure. 5917 */ 5918 static int 5919 _base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5920 { 5921 /* reply pool, 4 byte align */ 5922 ioc->reply_dma_pool = dma_pool_create("reply pool", 5923 &ioc->pdev->dev, sz, 4, 0); 5924 if (!ioc->reply_dma_pool) 5925 return -ENOMEM; 5926 ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL, 5927 &ioc->reply_dma); 5928 if (!ioc->reply) 5929 return -EAGAIN; 5930 if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) { 5931 dinitprintk(ioc, pr_err( 5932 "Bad Reply Pool! Reply (0x%p) Reply dma = (0x%llx)\n", 5933 ioc->reply, (unsigned long long) ioc->reply_dma)); 5934 ioc->use_32bit_dma = true; 5935 return -EAGAIN; 5936 } 5937 ioc->reply_dma_min_address = (u32)(ioc->reply_dma); 5938 ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz; 5939 ioc_info(ioc, 5940 "reply pool(0x%p) - dma(0x%llx): depth(%d), frame_size(%d), pool_size(%d kB)\n", 5941 ioc->reply, (unsigned long long)ioc->reply_dma, 5942 ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024); 5943 return 0; 5944 } 5945 5946 /** 5947 * _base_allocate_reply_free_dma_pool - Allocating DMA'able memory 5948 * for reply free dma pool. 5949 * @ioc: Adapter object 5950 * @sz: DMA Pool size 5951 * Return: 0 for success, non-zero for failure. 5952 */ 5953 static int 5954 _base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5955 { 5956 /* reply free queue, 16 byte align */ 5957 ioc->reply_free_dma_pool = dma_pool_create( 5958 "reply_free pool", &ioc->pdev->dev, sz, 16, 0); 5959 if (!ioc->reply_free_dma_pool) 5960 return -ENOMEM; 5961 ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, 5962 GFP_KERNEL, &ioc->reply_free_dma); 5963 if (!ioc->reply_free) 5964 return -EAGAIN; 5965 if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) { 5966 dinitprintk(ioc, 5967 pr_err("Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n", 5968 ioc->reply_free, (unsigned long long) ioc->reply_free_dma)); 5969 ioc->use_32bit_dma = true; 5970 return -EAGAIN; 5971 } 5972 memset(ioc->reply_free, 0, sz); 5973 dinitprintk(ioc, ioc_info(ioc, 5974 "reply_free pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n", 5975 ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024)); 5976 dinitprintk(ioc, ioc_info(ioc, 5977 "reply_free_dma (0x%llx)\n", 5978 (unsigned long long)ioc->reply_free_dma)); 5979 return 0; 5980 } 5981 5982 /** 5983 * _base_allocate_reply_post_free_array - Allocating DMA'able memory 5984 * for reply post free array. 5985 * @ioc: Adapter object 5986 * @reply_post_free_array_sz: DMA Pool size 5987 * Return: 0 for success, non-zero for failure. 5988 */ 5989 5990 static int 5991 _base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc, 5992 u32 reply_post_free_array_sz) 5993 { 5994 ioc->reply_post_free_array_dma_pool = 5995 dma_pool_create("reply_post_free_array pool", 5996 &ioc->pdev->dev, reply_post_free_array_sz, 16, 0); 5997 if (!ioc->reply_post_free_array_dma_pool) 5998 return -ENOMEM; 5999 ioc->reply_post_free_array = 6000 dma_pool_alloc(ioc->reply_post_free_array_dma_pool, 6001 GFP_KERNEL, &ioc->reply_post_free_array_dma); 6002 if (!ioc->reply_post_free_array) 6003 return -EAGAIN; 6004 if (!mpt3sas_check_same_4gb_region((long)ioc->reply_post_free_array, 6005 reply_post_free_array_sz)) { 6006 dinitprintk(ioc, pr_err( 6007 "Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n", 6008 ioc->reply_free, 6009 (unsigned long long) ioc->reply_free_dma)); 6010 ioc->use_32bit_dma = true; 6011 return -EAGAIN; 6012 } 6013 return 0; 6014 } 6015 /** 6016 * base_alloc_rdpq_dma_pool - Allocating DMA'able memory 6017 * for reply queues. 6018 * @ioc: per adapter object 6019 * @sz: DMA Pool size 6020 * Return: 0 for success, non-zero for failure. 6021 */ 6022 static int 6023 base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz) 6024 { 6025 int i = 0; 6026 u32 dma_alloc_count = 0; 6027 int reply_post_free_sz = ioc->reply_post_queue_depth * 6028 sizeof(Mpi2DefaultReplyDescriptor_t); 6029 int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1; 6030 6031 ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct), 6032 GFP_KERNEL); 6033 if (!ioc->reply_post) 6034 return -ENOMEM; 6035 /* 6036 * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and 6037 * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should 6038 * be within 4GB boundary i.e reply queues in a set must have same 6039 * upper 32-bits in their memory address. so here driver is allocating 6040 * the DMA'able memory for reply queues according. 6041 * Driver uses limitation of 6042 * VENTURA_SERIES to manage INVADER_SERIES as well. 6043 */ 6044 dma_alloc_count = DIV_ROUND_UP(count, 6045 RDPQ_MAX_INDEX_IN_ONE_CHUNK); 6046 ioc->reply_post_free_dma_pool = 6047 dma_pool_create("reply_post_free pool", 6048 &ioc->pdev->dev, sz, 16, 0); 6049 if (!ioc->reply_post_free_dma_pool) 6050 return -ENOMEM; 6051 for (i = 0; i < count; i++) { 6052 if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) { 6053 ioc->reply_post[i].reply_post_free = 6054 dma_pool_zalloc(ioc->reply_post_free_dma_pool, 6055 GFP_KERNEL, 6056 &ioc->reply_post[i].reply_post_free_dma); 6057 if (!ioc->reply_post[i].reply_post_free) 6058 return -ENOMEM; 6059 /* 6060 * Each set of RDPQ pool must satisfy 4gb boundary 6061 * restriction. 6062 * 1) Check if allocated resources for RDPQ pool are in 6063 * the same 4GB range. 6064 * 2) If #1 is true, continue with 64 bit DMA. 6065 * 3) If #1 is false, return 1. which means free all the 6066 * resources and set DMA mask to 32 and allocate. 6067 */ 6068 if (!mpt3sas_check_same_4gb_region( 6069 (long)ioc->reply_post[i].reply_post_free, sz)) { 6070 dinitprintk(ioc, 6071 ioc_err(ioc, "bad Replypost free pool(0x%p)" 6072 "reply_post_free_dma = (0x%llx)\n", 6073 ioc->reply_post[i].reply_post_free, 6074 (unsigned long long) 6075 ioc->reply_post[i].reply_post_free_dma)); 6076 return -EAGAIN; 6077 } 6078 dma_alloc_count--; 6079 6080 } else { 6081 ioc->reply_post[i].reply_post_free = 6082 (Mpi2ReplyDescriptorsUnion_t *) 6083 ((long)ioc->reply_post[i-1].reply_post_free 6084 + reply_post_free_sz); 6085 ioc->reply_post[i].reply_post_free_dma = 6086 (dma_addr_t) 6087 (ioc->reply_post[i-1].reply_post_free_dma + 6088 reply_post_free_sz); 6089 } 6090 } 6091 return 0; 6092 } 6093 6094 /** 6095 * _base_allocate_memory_pools - allocate start of day memory pools 6096 * @ioc: per adapter object 6097 * 6098 * Return: 0 success, anything else error. 6099 */ 6100 static int 6101 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) 6102 { 6103 struct mpt3sas_facts *facts; 6104 u16 max_sge_elements; 6105 u16 chains_needed_per_io; 6106 u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz; 6107 u32 retry_sz; 6108 u32 rdpq_sz = 0, sense_sz = 0; 6109 u16 max_request_credit, nvme_blocks_needed; 6110 unsigned short sg_tablesize; 6111 u16 sge_size; 6112 int i; 6113 int ret = 0, rc = 0; 6114 6115 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 6116 6117 6118 retry_sz = 0; 6119 facts = &ioc->facts; 6120 6121 /* command line tunables for max sgl entries */ 6122 if (max_sgl_entries != -1) 6123 sg_tablesize = max_sgl_entries; 6124 else { 6125 if (ioc->hba_mpi_version_belonged == MPI2_VERSION) 6126 sg_tablesize = MPT2SAS_SG_DEPTH; 6127 else 6128 sg_tablesize = MPT3SAS_SG_DEPTH; 6129 } 6130 6131 /* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */ 6132 if (reset_devices) 6133 sg_tablesize = min_t(unsigned short, sg_tablesize, 6134 MPT_KDUMP_MIN_PHYS_SEGMENTS); 6135 6136 if (ioc->is_mcpu_endpoint) 6137 ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS; 6138 else { 6139 if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS) 6140 sg_tablesize = MPT_MIN_PHYS_SEGMENTS; 6141 else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) { 6142 sg_tablesize = min_t(unsigned short, sg_tablesize, 6143 SG_MAX_SEGMENTS); 6144 ioc_warn(ioc, "sg_tablesize(%u) is bigger than kernel defined SG_CHUNK_SIZE(%u)\n", 6145 sg_tablesize, MPT_MAX_PHYS_SEGMENTS); 6146 } 6147 ioc->shost->sg_tablesize = sg_tablesize; 6148 } 6149 6150 ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)), 6151 (facts->RequestCredit / 4)); 6152 if (ioc->internal_depth < INTERNAL_CMDS_COUNT) { 6153 if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT + 6154 INTERNAL_SCSIIO_CMDS_COUNT)) { 6155 ioc_err(ioc, "IOC doesn't have enough Request Credits, it has just %d number of credits\n", 6156 facts->RequestCredit); 6157 return -ENOMEM; 6158 } 6159 ioc->internal_depth = 10; 6160 } 6161 6162 ioc->hi_priority_depth = ioc->internal_depth - (5); 6163 /* command line tunables for max controller queue depth */ 6164 if (max_queue_depth != -1 && max_queue_depth != 0) { 6165 max_request_credit = min_t(u16, max_queue_depth + 6166 ioc->internal_depth, facts->RequestCredit); 6167 if (max_request_credit > MAX_HBA_QUEUE_DEPTH) 6168 max_request_credit = MAX_HBA_QUEUE_DEPTH; 6169 } else if (reset_devices) 6170 max_request_credit = min_t(u16, facts->RequestCredit, 6171 (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth)); 6172 else 6173 max_request_credit = min_t(u16, facts->RequestCredit, 6174 MAX_HBA_QUEUE_DEPTH); 6175 6176 /* Firmware maintains additional facts->HighPriorityCredit number of 6177 * credits for HiPriprity Request messages, so hba queue depth will be 6178 * sum of max_request_credit and high priority queue depth. 6179 */ 6180 ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth; 6181 6182 /* request frame size */ 6183 ioc->request_sz = facts->IOCRequestFrameSize * 4; 6184 6185 /* reply frame size */ 6186 ioc->reply_sz = facts->ReplyFrameSize * 4; 6187 6188 /* chain segment size */ 6189 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { 6190 if (facts->IOCMaxChainSegmentSize) 6191 ioc->chain_segment_sz = 6192 facts->IOCMaxChainSegmentSize * 6193 MAX_CHAIN_ELEMT_SZ; 6194 else 6195 /* set to 128 bytes size if IOCMaxChainSegmentSize is zero */ 6196 ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS * 6197 MAX_CHAIN_ELEMT_SZ; 6198 } else 6199 ioc->chain_segment_sz = ioc->request_sz; 6200 6201 /* calculate the max scatter element size */ 6202 sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee); 6203 6204 retry_allocation: 6205 total_sz = 0; 6206 /* calculate number of sg elements left over in the 1st frame */ 6207 max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) - 6208 sizeof(Mpi2SGEIOUnion_t)) + sge_size); 6209 ioc->max_sges_in_main_message = max_sge_elements/sge_size; 6210 6211 /* now do the same for a chain buffer */ 6212 max_sge_elements = ioc->chain_segment_sz - sge_size; 6213 ioc->max_sges_in_chain_message = max_sge_elements/sge_size; 6214 6215 /* 6216 * MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE 6217 */ 6218 chains_needed_per_io = ((ioc->shost->sg_tablesize - 6219 ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message) 6220 + 1; 6221 if (chains_needed_per_io > facts->MaxChainDepth) { 6222 chains_needed_per_io = facts->MaxChainDepth; 6223 ioc->shost->sg_tablesize = min_t(u16, 6224 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message 6225 * chains_needed_per_io), ioc->shost->sg_tablesize); 6226 } 6227 ioc->chains_needed_per_io = chains_needed_per_io; 6228 6229 /* reply free queue sizing - taking into account for 64 FW events */ 6230 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64; 6231 6232 /* mCPU manage single counters for simplicity */ 6233 if (ioc->is_mcpu_endpoint) 6234 ioc->reply_post_queue_depth = ioc->reply_free_queue_depth; 6235 else { 6236 /* calculate reply descriptor post queue depth */ 6237 ioc->reply_post_queue_depth = ioc->hba_queue_depth + 6238 ioc->reply_free_queue_depth + 1; 6239 /* align the reply post queue on the next 16 count boundary */ 6240 if (ioc->reply_post_queue_depth % 16) 6241 ioc->reply_post_queue_depth += 16 - 6242 (ioc->reply_post_queue_depth % 16); 6243 } 6244 6245 if (ioc->reply_post_queue_depth > 6246 facts->MaxReplyDescriptorPostQueueDepth) { 6247 ioc->reply_post_queue_depth = 6248 facts->MaxReplyDescriptorPostQueueDepth - 6249 (facts->MaxReplyDescriptorPostQueueDepth % 16); 6250 ioc->hba_queue_depth = 6251 ((ioc->reply_post_queue_depth - 64) / 2) - 1; 6252 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64; 6253 } 6254 6255 ioc_info(ioc, 6256 "scatter gather: sge_in_main_msg(%d), sge_per_chain(%d), " 6257 "sge_per_io(%d), chains_per_io(%d)\n", 6258 ioc->max_sges_in_main_message, 6259 ioc->max_sges_in_chain_message, 6260 ioc->shost->sg_tablesize, 6261 ioc->chains_needed_per_io); 6262 6263 /* reply post queue, 16 byte align */ 6264 reply_post_free_sz = ioc->reply_post_queue_depth * 6265 sizeof(Mpi2DefaultReplyDescriptor_t); 6266 rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK; 6267 if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable) 6268 || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK)) 6269 rdpq_sz = reply_post_free_sz * ioc->reply_queue_count; 6270 ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz); 6271 if (ret == -EAGAIN) { 6272 /* 6273 * Free allocated bad RDPQ memory pools. 6274 * Change dma coherent mask to 32 bit and reallocate RDPQ 6275 */ 6276 _base_release_memory_pools(ioc); 6277 ioc->use_32bit_dma = true; 6278 if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) { 6279 ioc_err(ioc, 6280 "32 DMA mask failed %s\n", pci_name(ioc->pdev)); 6281 return -ENODEV; 6282 } 6283 if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz)) 6284 return -ENOMEM; 6285 } else if (ret == -ENOMEM) 6286 return -ENOMEM; 6287 total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 : 6288 DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK)); 6289 ioc->scsiio_depth = ioc->hba_queue_depth - 6290 ioc->hi_priority_depth - ioc->internal_depth; 6291 6292 /* set the scsi host can_queue depth 6293 * with some internal commands that could be outstanding 6294 */ 6295 ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT; 6296 dinitprintk(ioc, 6297 ioc_info(ioc, "scsi host: can_queue depth (%d)\n", 6298 ioc->shost->can_queue)); 6299 6300 /* contiguous pool for request and chains, 16 byte align, one extra " 6301 * "frame for smid=0 6302 */ 6303 ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth; 6304 sz = ((ioc->scsiio_depth + 1) * ioc->request_sz); 6305 6306 /* hi-priority queue */ 6307 sz += (ioc->hi_priority_depth * ioc->request_sz); 6308 6309 /* internal queue */ 6310 sz += (ioc->internal_depth * ioc->request_sz); 6311 6312 ioc->request_dma_sz = sz; 6313 ioc->request = dma_alloc_coherent(&ioc->pdev->dev, sz, 6314 &ioc->request_dma, GFP_KERNEL); 6315 if (!ioc->request) { 6316 ioc_err(ioc, "request pool: dma_alloc_coherent failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kB)\n", 6317 ioc->hba_queue_depth, ioc->chains_needed_per_io, 6318 ioc->request_sz, sz / 1024); 6319 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH) 6320 goto out; 6321 retry_sz = 64; 6322 ioc->hba_queue_depth -= retry_sz; 6323 _base_release_memory_pools(ioc); 6324 goto retry_allocation; 6325 } 6326 6327 if (retry_sz) 6328 ioc_err(ioc, "request pool: dma_alloc_coherent succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kb)\n", 6329 ioc->hba_queue_depth, ioc->chains_needed_per_io, 6330 ioc->request_sz, sz / 1024); 6331 6332 /* hi-priority queue */ 6333 ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) * 6334 ioc->request_sz); 6335 ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) * 6336 ioc->request_sz); 6337 6338 /* internal queue */ 6339 ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth * 6340 ioc->request_sz); 6341 ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth * 6342 ioc->request_sz); 6343 6344 ioc_info(ioc, 6345 "request pool(0x%p) - dma(0x%llx): " 6346 "depth(%d), frame_size(%d), pool_size(%d kB)\n", 6347 ioc->request, (unsigned long long) ioc->request_dma, 6348 ioc->hba_queue_depth, ioc->request_sz, 6349 (ioc->hba_queue_depth * ioc->request_sz) / 1024); 6350 6351 total_sz += sz; 6352 6353 dinitprintk(ioc, 6354 ioc_info(ioc, "scsiio(0x%p): depth(%d)\n", 6355 ioc->request, ioc->scsiio_depth)); 6356 6357 ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH); 6358 sz = ioc->scsiio_depth * sizeof(struct chain_lookup); 6359 ioc->chain_lookup = kzalloc(sz, GFP_KERNEL); 6360 if (!ioc->chain_lookup) { 6361 ioc_err(ioc, "chain_lookup: __get_free_pages failed\n"); 6362 goto out; 6363 } 6364 6365 sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker); 6366 for (i = 0; i < ioc->scsiio_depth; i++) { 6367 ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL); 6368 if (!ioc->chain_lookup[i].chains_per_smid) { 6369 ioc_err(ioc, "chain_lookup: kzalloc failed\n"); 6370 goto out; 6371 } 6372 } 6373 6374 /* initialize hi-priority queue smid's */ 6375 ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth, 6376 sizeof(struct request_tracker), GFP_KERNEL); 6377 if (!ioc->hpr_lookup) { 6378 ioc_err(ioc, "hpr_lookup: kcalloc failed\n"); 6379 goto out; 6380 } 6381 ioc->hi_priority_smid = ioc->scsiio_depth + 1; 6382 dinitprintk(ioc, 6383 ioc_info(ioc, "hi_priority(0x%p): depth(%d), start smid(%d)\n", 6384 ioc->hi_priority, 6385 ioc->hi_priority_depth, ioc->hi_priority_smid)); 6386 6387 /* initialize internal queue smid's */ 6388 ioc->internal_lookup = kcalloc(ioc->internal_depth, 6389 sizeof(struct request_tracker), GFP_KERNEL); 6390 if (!ioc->internal_lookup) { 6391 ioc_err(ioc, "internal_lookup: kcalloc failed\n"); 6392 goto out; 6393 } 6394 ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth; 6395 dinitprintk(ioc, 6396 ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n", 6397 ioc->internal, 6398 ioc->internal_depth, ioc->internal_smid)); 6399 6400 ioc->io_queue_num = kcalloc(ioc->scsiio_depth, 6401 sizeof(u16), GFP_KERNEL); 6402 if (!ioc->io_queue_num) 6403 goto out; 6404 /* 6405 * The number of NVMe page sized blocks needed is: 6406 * (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1 6407 * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry 6408 * that is placed in the main message frame. 8 is the size of each PRP 6409 * entry or PRP list pointer entry. 8 is subtracted from page_size 6410 * because of the PRP list pointer entry at the end of a page, so this 6411 * is not counted as a PRP entry. The 1 added page is a round up. 6412 * 6413 * To avoid allocation failures due to the amount of memory that could 6414 * be required for NVMe PRP's, only each set of NVMe blocks will be 6415 * contiguous, so a new set is allocated for each possible I/O. 6416 */ 6417 6418 ioc->chains_per_prp_buffer = 0; 6419 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) { 6420 nvme_blocks_needed = 6421 (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1; 6422 nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE); 6423 nvme_blocks_needed++; 6424 6425 sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth; 6426 ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL); 6427 if (!ioc->pcie_sg_lookup) { 6428 ioc_info(ioc, "PCIe SGL lookup: kzalloc failed\n"); 6429 goto out; 6430 } 6431 sz = nvme_blocks_needed * ioc->page_size; 6432 rc = _base_allocate_pcie_sgl_pool(ioc, sz); 6433 if (rc == -ENOMEM) 6434 return -ENOMEM; 6435 else if (rc == -EAGAIN) 6436 goto try_32bit_dma; 6437 total_sz += sz * ioc->scsiio_depth; 6438 } 6439 6440 rc = _base_allocate_chain_dma_pool(ioc, ioc->chain_segment_sz); 6441 if (rc == -ENOMEM) 6442 return -ENOMEM; 6443 else if (rc == -EAGAIN) 6444 goto try_32bit_dma; 6445 total_sz += ioc->chain_segment_sz * ((ioc->chains_needed_per_io - 6446 ioc->chains_per_prp_buffer) * ioc->scsiio_depth); 6447 dinitprintk(ioc, 6448 ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n", 6449 ioc->chain_depth, ioc->chain_segment_sz, 6450 (ioc->chain_depth * ioc->chain_segment_sz) / 1024)); 6451 /* sense buffers, 4 byte align */ 6452 sense_sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE; 6453 rc = _base_allocate_sense_dma_pool(ioc, sense_sz); 6454 if (rc == -ENOMEM) 6455 return -ENOMEM; 6456 else if (rc == -EAGAIN) 6457 goto try_32bit_dma; 6458 total_sz += sense_sz; 6459 ioc_info(ioc, 6460 "sense pool(0x%p)- dma(0x%llx): depth(%d)," 6461 "element_size(%d), pool_size(%d kB)\n", 6462 ioc->sense, (unsigned long long)ioc->sense_dma, ioc->scsiio_depth, 6463 SCSI_SENSE_BUFFERSIZE, sz / 1024); 6464 /* reply pool, 4 byte align */ 6465 sz = ioc->reply_free_queue_depth * ioc->reply_sz; 6466 rc = _base_allocate_reply_pool(ioc, sz); 6467 if (rc == -ENOMEM) 6468 return -ENOMEM; 6469 else if (rc == -EAGAIN) 6470 goto try_32bit_dma; 6471 total_sz += sz; 6472 6473 /* reply free queue, 16 byte align */ 6474 sz = ioc->reply_free_queue_depth * 4; 6475 rc = _base_allocate_reply_free_dma_pool(ioc, sz); 6476 if (rc == -ENOMEM) 6477 return -ENOMEM; 6478 else if (rc == -EAGAIN) 6479 goto try_32bit_dma; 6480 dinitprintk(ioc, 6481 ioc_info(ioc, "reply_free_dma (0x%llx)\n", 6482 (unsigned long long)ioc->reply_free_dma)); 6483 total_sz += sz; 6484 if (ioc->rdpq_array_enable) { 6485 reply_post_free_array_sz = ioc->reply_queue_count * 6486 sizeof(Mpi2IOCInitRDPQArrayEntry); 6487 rc = _base_allocate_reply_post_free_array(ioc, 6488 reply_post_free_array_sz); 6489 if (rc == -ENOMEM) 6490 return -ENOMEM; 6491 else if (rc == -EAGAIN) 6492 goto try_32bit_dma; 6493 } 6494 ioc->config_page_sz = 512; 6495 ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev, 6496 ioc->config_page_sz, &ioc->config_page_dma, GFP_KERNEL); 6497 if (!ioc->config_page) { 6498 ioc_err(ioc, "config page: dma_pool_alloc failed\n"); 6499 goto out; 6500 } 6501 6502 ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n", 6503 ioc->config_page, (unsigned long long)ioc->config_page_dma, 6504 ioc->config_page_sz); 6505 total_sz += ioc->config_page_sz; 6506 6507 ioc_info(ioc, "Allocated physical memory: size(%d kB)\n", 6508 total_sz / 1024); 6509 ioc_info(ioc, "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n", 6510 ioc->shost->can_queue, facts->RequestCredit); 6511 ioc_info(ioc, "Scatter Gather Elements per IO(%d)\n", 6512 ioc->shost->sg_tablesize); 6513 return 0; 6514 6515 try_32bit_dma: 6516 _base_release_memory_pools(ioc); 6517 if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) { 6518 /* Change dma coherent mask to 32 bit and reallocate */ 6519 if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) { 6520 pr_err("Setting 32 bit coherent DMA mask Failed %s\n", 6521 pci_name(ioc->pdev)); 6522 return -ENODEV; 6523 } 6524 } else if (_base_reduce_hba_queue_depth(ioc) != 0) 6525 return -ENOMEM; 6526 goto retry_allocation; 6527 6528 out: 6529 return -ENOMEM; 6530 } 6531 6532 /** 6533 * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter. 6534 * @ioc: Pointer to MPT_ADAPTER structure 6535 * @cooked: Request raw or cooked IOC state 6536 * 6537 * Return: all IOC Doorbell register bits if cooked==0, else just the 6538 * Doorbell bits in MPI_IOC_STATE_MASK. 6539 */ 6540 u32 6541 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked) 6542 { 6543 u32 s, sc; 6544 6545 s = ioc->base_readl(&ioc->chip->Doorbell); 6546 sc = s & MPI2_IOC_STATE_MASK; 6547 return cooked ? sc : s; 6548 } 6549 6550 /** 6551 * _base_wait_on_iocstate - waiting on a particular ioc state 6552 * @ioc: ? 6553 * @ioc_state: controller state { READY, OPERATIONAL, or RESET } 6554 * @timeout: timeout in second 6555 * 6556 * Return: 0 for success, non-zero for failure. 6557 */ 6558 static int 6559 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout) 6560 { 6561 u32 count, cntdn; 6562 u32 current_state; 6563 6564 count = 0; 6565 cntdn = 1000 * timeout; 6566 do { 6567 current_state = mpt3sas_base_get_iocstate(ioc, 1); 6568 if (current_state == ioc_state) 6569 return 0; 6570 if (count && current_state == MPI2_IOC_STATE_FAULT) 6571 break; 6572 if (count && current_state == MPI2_IOC_STATE_COREDUMP) 6573 break; 6574 6575 usleep_range(1000, 1500); 6576 count++; 6577 } while (--cntdn); 6578 6579 return current_state; 6580 } 6581 6582 /** 6583 * _base_dump_reg_set - This function will print hexdump of register set. 6584 * @ioc: per adapter object 6585 * 6586 * Return: nothing. 6587 */ 6588 static inline void 6589 _base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc) 6590 { 6591 unsigned int i, sz = 256; 6592 u32 __iomem *reg = (u32 __iomem *)ioc->chip; 6593 6594 ioc_info(ioc, "System Register set:\n"); 6595 for (i = 0; i < (sz / sizeof(u32)); i++) 6596 pr_info("%08x: %08x\n", (i * 4), readl(®[i])); 6597 } 6598 6599 /** 6600 * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by 6601 * a write to the doorbell) 6602 * @ioc: per adapter object 6603 * @timeout: timeout in seconds 6604 * 6605 * Return: 0 for success, non-zero for failure. 6606 * 6607 * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell. 6608 */ 6609 6610 static int 6611 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout) 6612 { 6613 u32 cntdn, count; 6614 u32 int_status; 6615 6616 count = 0; 6617 cntdn = 1000 * timeout; 6618 do { 6619 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus); 6620 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 6621 dhsprintk(ioc, 6622 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6623 __func__, count, timeout)); 6624 return 0; 6625 } 6626 6627 usleep_range(1000, 1500); 6628 count++; 6629 } while (--cntdn); 6630 6631 ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n", 6632 __func__, count, int_status); 6633 return -EFAULT; 6634 } 6635 6636 static int 6637 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout) 6638 { 6639 u32 cntdn, count; 6640 u32 int_status; 6641 6642 count = 0; 6643 cntdn = 2000 * timeout; 6644 do { 6645 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus); 6646 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 6647 dhsprintk(ioc, 6648 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6649 __func__, count, timeout)); 6650 return 0; 6651 } 6652 6653 udelay(500); 6654 count++; 6655 } while (--cntdn); 6656 6657 ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n", 6658 __func__, count, int_status); 6659 return -EFAULT; 6660 6661 } 6662 6663 /** 6664 * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell. 6665 * @ioc: per adapter object 6666 * @timeout: timeout in second 6667 * 6668 * Return: 0 for success, non-zero for failure. 6669 * 6670 * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to 6671 * doorbell. 6672 */ 6673 static int 6674 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout) 6675 { 6676 u32 cntdn, count; 6677 u32 int_status; 6678 u32 doorbell; 6679 6680 count = 0; 6681 cntdn = 1000 * timeout; 6682 do { 6683 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus); 6684 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) { 6685 dhsprintk(ioc, 6686 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6687 __func__, count, timeout)); 6688 return 0; 6689 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 6690 doorbell = ioc->base_readl(&ioc->chip->Doorbell); 6691 if ((doorbell & MPI2_IOC_STATE_MASK) == 6692 MPI2_IOC_STATE_FAULT) { 6693 mpt3sas_print_fault_code(ioc, doorbell); 6694 return -EFAULT; 6695 } 6696 if ((doorbell & MPI2_IOC_STATE_MASK) == 6697 MPI2_IOC_STATE_COREDUMP) { 6698 mpt3sas_print_coredump_info(ioc, doorbell); 6699 return -EFAULT; 6700 } 6701 } else if (int_status == 0xFFFFFFFF) 6702 goto out; 6703 6704 usleep_range(1000, 1500); 6705 count++; 6706 } while (--cntdn); 6707 6708 out: 6709 ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n", 6710 __func__, count, int_status); 6711 return -EFAULT; 6712 } 6713 6714 /** 6715 * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use 6716 * @ioc: per adapter object 6717 * @timeout: timeout in second 6718 * 6719 * Return: 0 for success, non-zero for failure. 6720 */ 6721 static int 6722 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout) 6723 { 6724 u32 cntdn, count; 6725 u32 doorbell_reg; 6726 6727 count = 0; 6728 cntdn = 1000 * timeout; 6729 do { 6730 doorbell_reg = ioc->base_readl(&ioc->chip->Doorbell); 6731 if (!(doorbell_reg & MPI2_DOORBELL_USED)) { 6732 dhsprintk(ioc, 6733 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6734 __func__, count, timeout)); 6735 return 0; 6736 } 6737 6738 usleep_range(1000, 1500); 6739 count++; 6740 } while (--cntdn); 6741 6742 ioc_err(ioc, "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n", 6743 __func__, count, doorbell_reg); 6744 return -EFAULT; 6745 } 6746 6747 /** 6748 * _base_send_ioc_reset - send doorbell reset 6749 * @ioc: per adapter object 6750 * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET 6751 * @timeout: timeout in second 6752 * 6753 * Return: 0 for success, non-zero for failure. 6754 */ 6755 static int 6756 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout) 6757 { 6758 u32 ioc_state; 6759 int r = 0; 6760 unsigned long flags; 6761 6762 if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) { 6763 ioc_err(ioc, "%s: unknown reset_type\n", __func__); 6764 return -EFAULT; 6765 } 6766 6767 if (!(ioc->facts.IOCCapabilities & 6768 MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY)) 6769 return -EFAULT; 6770 6771 ioc_info(ioc, "sending message unit reset !!\n"); 6772 6773 writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT, 6774 &ioc->chip->Doorbell); 6775 if ((_base_wait_for_doorbell_ack(ioc, 15))) { 6776 r = -EFAULT; 6777 goto out; 6778 } 6779 6780 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout); 6781 if (ioc_state) { 6782 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 6783 __func__, ioc_state); 6784 r = -EFAULT; 6785 goto out; 6786 } 6787 out: 6788 if (r != 0) { 6789 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 6790 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 6791 /* 6792 * Wait for IOC state CoreDump to clear only during 6793 * HBA initialization & release time. 6794 */ 6795 if ((ioc_state & MPI2_IOC_STATE_MASK) == 6796 MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 || 6797 ioc->fault_reset_work_q == NULL)) { 6798 spin_unlock_irqrestore( 6799 &ioc->ioc_reset_in_progress_lock, flags); 6800 mpt3sas_print_coredump_info(ioc, ioc_state); 6801 mpt3sas_base_wait_for_coredump_completion(ioc, 6802 __func__); 6803 spin_lock_irqsave( 6804 &ioc->ioc_reset_in_progress_lock, flags); 6805 } 6806 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 6807 } 6808 ioc_info(ioc, "message unit reset: %s\n", 6809 r == 0 ? "SUCCESS" : "FAILED"); 6810 return r; 6811 } 6812 6813 /** 6814 * mpt3sas_wait_for_ioc - IOC's operational state is checked here. 6815 * @ioc: per adapter object 6816 * @timeout: timeout in seconds 6817 * 6818 * Return: Waits up to timeout seconds for the IOC to 6819 * become operational. Returns 0 if IOC is present 6820 * and operational; otherwise returns %-EFAULT. 6821 */ 6822 6823 int 6824 mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout) 6825 { 6826 int wait_state_count = 0; 6827 u32 ioc_state; 6828 6829 do { 6830 ioc_state = mpt3sas_base_get_iocstate(ioc, 1); 6831 if (ioc_state == MPI2_IOC_STATE_OPERATIONAL) 6832 break; 6833 6834 /* 6835 * Watchdog thread will be started after IOC Initialization, so 6836 * no need to wait here for IOC state to become operational 6837 * when IOC Initialization is on. Instead the driver will 6838 * return ETIME status, so that calling function can issue 6839 * diag reset operation and retry the command. 6840 */ 6841 if (ioc->is_driver_loading) 6842 return -ETIME; 6843 6844 ssleep(1); 6845 ioc_info(ioc, "%s: waiting for operational state(count=%d)\n", 6846 __func__, ++wait_state_count); 6847 } while (--timeout); 6848 if (!timeout) { 6849 ioc_err(ioc, "%s: failed due to ioc not operational\n", __func__); 6850 return -EFAULT; 6851 } 6852 if (wait_state_count) 6853 ioc_info(ioc, "ioc is operational\n"); 6854 return 0; 6855 } 6856 6857 /** 6858 * _base_handshake_req_reply_wait - send request thru doorbell interface 6859 * @ioc: per adapter object 6860 * @request_bytes: request length 6861 * @request: pointer having request payload 6862 * @reply_bytes: reply length 6863 * @reply: pointer to reply payload 6864 * @timeout: timeout in second 6865 * 6866 * Return: 0 for success, non-zero for failure. 6867 */ 6868 static int 6869 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, 6870 u32 *request, int reply_bytes, u16 *reply, int timeout) 6871 { 6872 MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply; 6873 int i; 6874 u8 failed; 6875 __le32 *mfp; 6876 6877 /* make sure doorbell is not in use */ 6878 if ((ioc->base_readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) { 6879 ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__); 6880 return -EFAULT; 6881 } 6882 6883 /* clear pending doorbell interrupts from previous state changes */ 6884 if (ioc->base_readl(&ioc->chip->HostInterruptStatus) & 6885 MPI2_HIS_IOC2SYS_DB_STATUS) 6886 writel(0, &ioc->chip->HostInterruptStatus); 6887 6888 /* send message to ioc */ 6889 writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) | 6890 ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)), 6891 &ioc->chip->Doorbell); 6892 6893 if ((_base_spin_on_doorbell_int(ioc, 5))) { 6894 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6895 __LINE__); 6896 return -EFAULT; 6897 } 6898 writel(0, &ioc->chip->HostInterruptStatus); 6899 6900 if ((_base_wait_for_doorbell_ack(ioc, 5))) { 6901 ioc_err(ioc, "doorbell handshake ack failed (line=%d)\n", 6902 __LINE__); 6903 return -EFAULT; 6904 } 6905 6906 /* send message 32-bits at a time */ 6907 for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) { 6908 writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell); 6909 if ((_base_wait_for_doorbell_ack(ioc, 5))) 6910 failed = 1; 6911 } 6912 6913 if (failed) { 6914 ioc_err(ioc, "doorbell handshake sending request failed (line=%d)\n", 6915 __LINE__); 6916 return -EFAULT; 6917 } 6918 6919 /* now wait for the reply */ 6920 if ((_base_wait_for_doorbell_int(ioc, timeout))) { 6921 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6922 __LINE__); 6923 return -EFAULT; 6924 } 6925 6926 /* read the first two 16-bits, it gives the total length of the reply */ 6927 reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) 6928 & MPI2_DOORBELL_DATA_MASK); 6929 writel(0, &ioc->chip->HostInterruptStatus); 6930 if ((_base_wait_for_doorbell_int(ioc, 5))) { 6931 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6932 __LINE__); 6933 return -EFAULT; 6934 } 6935 reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) 6936 & MPI2_DOORBELL_DATA_MASK); 6937 writel(0, &ioc->chip->HostInterruptStatus); 6938 6939 for (i = 2; i < default_reply->MsgLength * 2; i++) { 6940 if ((_base_wait_for_doorbell_int(ioc, 5))) { 6941 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6942 __LINE__); 6943 return -EFAULT; 6944 } 6945 if (i >= reply_bytes/2) /* overflow case */ 6946 ioc->base_readl(&ioc->chip->Doorbell); 6947 else 6948 reply[i] = le16_to_cpu( 6949 ioc->base_readl(&ioc->chip->Doorbell) 6950 & MPI2_DOORBELL_DATA_MASK); 6951 writel(0, &ioc->chip->HostInterruptStatus); 6952 } 6953 6954 _base_wait_for_doorbell_int(ioc, 5); 6955 if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) { 6956 dhsprintk(ioc, 6957 ioc_info(ioc, "doorbell is in use (line=%d)\n", 6958 __LINE__)); 6959 } 6960 writel(0, &ioc->chip->HostInterruptStatus); 6961 6962 if (ioc->logging_level & MPT_DEBUG_INIT) { 6963 mfp = (__le32 *)reply; 6964 pr_info("\toffset:data\n"); 6965 for (i = 0; i < reply_bytes/4; i++) 6966 ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4, 6967 le32_to_cpu(mfp[i])); 6968 } 6969 return 0; 6970 } 6971 6972 /** 6973 * mpt3sas_base_sas_iounit_control - send sas iounit control to FW 6974 * @ioc: per adapter object 6975 * @mpi_reply: the reply payload from FW 6976 * @mpi_request: the request payload sent to FW 6977 * 6978 * The SAS IO Unit Control Request message allows the host to perform low-level 6979 * operations, such as resets on the PHYs of the IO Unit, also allows the host 6980 * to obtain the IOC assigned device handles for a device if it has other 6981 * identifying information about the device, in addition allows the host to 6982 * remove IOC resources associated with the device. 6983 * 6984 * Return: 0 for success, non-zero for failure. 6985 */ 6986 int 6987 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc, 6988 Mpi2SasIoUnitControlReply_t *mpi_reply, 6989 Mpi2SasIoUnitControlRequest_t *mpi_request) 6990 { 6991 u16 smid; 6992 u8 issue_reset = 0; 6993 int rc; 6994 void *request; 6995 6996 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 6997 6998 mutex_lock(&ioc->base_cmds.mutex); 6999 7000 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) { 7001 ioc_err(ioc, "%s: base_cmd in use\n", __func__); 7002 rc = -EAGAIN; 7003 goto out; 7004 } 7005 7006 rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT); 7007 if (rc) 7008 goto out; 7009 7010 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 7011 if (!smid) { 7012 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7013 rc = -EAGAIN; 7014 goto out; 7015 } 7016 7017 rc = 0; 7018 ioc->base_cmds.status = MPT3_CMD_PENDING; 7019 request = mpt3sas_base_get_msg_frame(ioc, smid); 7020 ioc->base_cmds.smid = smid; 7021 memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)); 7022 if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET || 7023 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) 7024 ioc->ioc_link_reset_in_progress = 1; 7025 init_completion(&ioc->base_cmds.done); 7026 ioc->put_smid_default(ioc, smid); 7027 wait_for_completion_timeout(&ioc->base_cmds.done, 7028 msecs_to_jiffies(10000)); 7029 if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET || 7030 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) && 7031 ioc->ioc_link_reset_in_progress) 7032 ioc->ioc_link_reset_in_progress = 0; 7033 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 7034 mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status, 7035 mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)/4, 7036 issue_reset); 7037 goto issue_host_reset; 7038 } 7039 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) 7040 memcpy(mpi_reply, ioc->base_cmds.reply, 7041 sizeof(Mpi2SasIoUnitControlReply_t)); 7042 else 7043 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t)); 7044 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7045 goto out; 7046 7047 issue_host_reset: 7048 if (issue_reset) 7049 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 7050 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7051 rc = -EFAULT; 7052 out: 7053 mutex_unlock(&ioc->base_cmds.mutex); 7054 return rc; 7055 } 7056 7057 /** 7058 * mpt3sas_base_scsi_enclosure_processor - sending request to sep device 7059 * @ioc: per adapter object 7060 * @mpi_reply: the reply payload from FW 7061 * @mpi_request: the request payload sent to FW 7062 * 7063 * The SCSI Enclosure Processor request message causes the IOC to 7064 * communicate with SES devices to control LED status signals. 7065 * 7066 * Return: 0 for success, non-zero for failure. 7067 */ 7068 int 7069 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc, 7070 Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request) 7071 { 7072 u16 smid; 7073 u8 issue_reset = 0; 7074 int rc; 7075 void *request; 7076 7077 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7078 7079 mutex_lock(&ioc->base_cmds.mutex); 7080 7081 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) { 7082 ioc_err(ioc, "%s: base_cmd in use\n", __func__); 7083 rc = -EAGAIN; 7084 goto out; 7085 } 7086 7087 rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT); 7088 if (rc) 7089 goto out; 7090 7091 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 7092 if (!smid) { 7093 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7094 rc = -EAGAIN; 7095 goto out; 7096 } 7097 7098 rc = 0; 7099 ioc->base_cmds.status = MPT3_CMD_PENDING; 7100 request = mpt3sas_base_get_msg_frame(ioc, smid); 7101 ioc->base_cmds.smid = smid; 7102 memset(request, 0, ioc->request_sz); 7103 memcpy(request, mpi_request, sizeof(Mpi2SepReply_t)); 7104 init_completion(&ioc->base_cmds.done); 7105 ioc->put_smid_default(ioc, smid); 7106 wait_for_completion_timeout(&ioc->base_cmds.done, 7107 msecs_to_jiffies(10000)); 7108 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 7109 mpt3sas_check_cmd_timeout(ioc, 7110 ioc->base_cmds.status, mpi_request, 7111 sizeof(Mpi2SepRequest_t)/4, issue_reset); 7112 goto issue_host_reset; 7113 } 7114 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) 7115 memcpy(mpi_reply, ioc->base_cmds.reply, 7116 sizeof(Mpi2SepReply_t)); 7117 else 7118 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t)); 7119 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7120 goto out; 7121 7122 issue_host_reset: 7123 if (issue_reset) 7124 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 7125 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7126 rc = -EFAULT; 7127 out: 7128 mutex_unlock(&ioc->base_cmds.mutex); 7129 return rc; 7130 } 7131 7132 /** 7133 * _base_get_port_facts - obtain port facts reply and save in ioc 7134 * @ioc: per adapter object 7135 * @port: ? 7136 * 7137 * Return: 0 for success, non-zero for failure. 7138 */ 7139 static int 7140 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port) 7141 { 7142 Mpi2PortFactsRequest_t mpi_request; 7143 Mpi2PortFactsReply_t mpi_reply; 7144 struct mpt3sas_port_facts *pfacts; 7145 int mpi_reply_sz, mpi_request_sz, r; 7146 7147 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7148 7149 mpi_reply_sz = sizeof(Mpi2PortFactsReply_t); 7150 mpi_request_sz = sizeof(Mpi2PortFactsRequest_t); 7151 memset(&mpi_request, 0, mpi_request_sz); 7152 mpi_request.Function = MPI2_FUNCTION_PORT_FACTS; 7153 mpi_request.PortNumber = port; 7154 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz, 7155 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5); 7156 7157 if (r != 0) { 7158 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); 7159 return r; 7160 } 7161 7162 pfacts = &ioc->pfacts[port]; 7163 memset(pfacts, 0, sizeof(struct mpt3sas_port_facts)); 7164 pfacts->PortNumber = mpi_reply.PortNumber; 7165 pfacts->VP_ID = mpi_reply.VP_ID; 7166 pfacts->VF_ID = mpi_reply.VF_ID; 7167 pfacts->MaxPostedCmdBuffers = 7168 le16_to_cpu(mpi_reply.MaxPostedCmdBuffers); 7169 7170 return 0; 7171 } 7172 7173 /** 7174 * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL 7175 * @ioc: per adapter object 7176 * @timeout: 7177 * 7178 * Return: 0 for success, non-zero for failure. 7179 */ 7180 static int 7181 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout) 7182 { 7183 u32 ioc_state; 7184 int rc; 7185 7186 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7187 7188 if (ioc->pci_error_recovery) { 7189 dfailprintk(ioc, 7190 ioc_info(ioc, "%s: host in pci error recovery\n", 7191 __func__)); 7192 return -EFAULT; 7193 } 7194 7195 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 7196 dhsprintk(ioc, 7197 ioc_info(ioc, "%s: ioc_state(0x%08x)\n", 7198 __func__, ioc_state)); 7199 7200 if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) || 7201 (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL) 7202 return 0; 7203 7204 if (ioc_state & MPI2_DOORBELL_USED) { 7205 dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n")); 7206 goto issue_diag_reset; 7207 } 7208 7209 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 7210 mpt3sas_print_fault_code(ioc, ioc_state & 7211 MPI2_DOORBELL_DATA_MASK); 7212 goto issue_diag_reset; 7213 } else if ((ioc_state & MPI2_IOC_STATE_MASK) == 7214 MPI2_IOC_STATE_COREDUMP) { 7215 ioc_info(ioc, 7216 "%s: Skipping the diag reset here. (ioc_state=0x%x)\n", 7217 __func__, ioc_state); 7218 return -EFAULT; 7219 } 7220 7221 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout); 7222 if (ioc_state) { 7223 dfailprintk(ioc, 7224 ioc_info(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 7225 __func__, ioc_state)); 7226 return -EFAULT; 7227 } 7228 7229 issue_diag_reset: 7230 rc = _base_diag_reset(ioc); 7231 return rc; 7232 } 7233 7234 /** 7235 * _base_get_ioc_facts - obtain ioc facts reply and save in ioc 7236 * @ioc: per adapter object 7237 * 7238 * Return: 0 for success, non-zero for failure. 7239 */ 7240 static int 7241 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc) 7242 { 7243 Mpi2IOCFactsRequest_t mpi_request; 7244 Mpi2IOCFactsReply_t mpi_reply; 7245 struct mpt3sas_facts *facts; 7246 int mpi_reply_sz, mpi_request_sz, r; 7247 7248 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7249 7250 r = _base_wait_for_iocstate(ioc, 10); 7251 if (r) { 7252 dfailprintk(ioc, 7253 ioc_info(ioc, "%s: failed getting to correct state\n", 7254 __func__)); 7255 return r; 7256 } 7257 mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t); 7258 mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t); 7259 memset(&mpi_request, 0, mpi_request_sz); 7260 mpi_request.Function = MPI2_FUNCTION_IOC_FACTS; 7261 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz, 7262 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5); 7263 7264 if (r != 0) { 7265 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); 7266 return r; 7267 } 7268 7269 facts = &ioc->facts; 7270 memset(facts, 0, sizeof(struct mpt3sas_facts)); 7271 facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion); 7272 facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion); 7273 facts->VP_ID = mpi_reply.VP_ID; 7274 facts->VF_ID = mpi_reply.VF_ID; 7275 facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions); 7276 facts->MaxChainDepth = mpi_reply.MaxChainDepth; 7277 facts->WhoInit = mpi_reply.WhoInit; 7278 facts->NumberOfPorts = mpi_reply.NumberOfPorts; 7279 facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors; 7280 if (ioc->msix_enable && (facts->MaxMSIxVectors <= 7281 MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc))) 7282 ioc->combined_reply_queue = 0; 7283 facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit); 7284 facts->MaxReplyDescriptorPostQueueDepth = 7285 le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth); 7286 facts->ProductID = le16_to_cpu(mpi_reply.ProductID); 7287 facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities); 7288 if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID)) 7289 ioc->ir_firmware = 1; 7290 if ((facts->IOCCapabilities & 7291 MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices)) 7292 ioc->rdpq_array_capable = 1; 7293 if ((facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ) 7294 && ioc->is_aero_ioc) 7295 ioc->atomic_desc_capable = 1; 7296 facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word); 7297 facts->IOCRequestFrameSize = 7298 le16_to_cpu(mpi_reply.IOCRequestFrameSize); 7299 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { 7300 facts->IOCMaxChainSegmentSize = 7301 le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize); 7302 } 7303 facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators); 7304 facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets); 7305 ioc->shost->max_id = -1; 7306 facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders); 7307 facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures); 7308 facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags); 7309 facts->HighPriorityCredit = 7310 le16_to_cpu(mpi_reply.HighPriorityCredit); 7311 facts->ReplyFrameSize = mpi_reply.ReplyFrameSize; 7312 facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle); 7313 facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize; 7314 7315 /* 7316 * Get the Page Size from IOC Facts. If it's 0, default to 4k. 7317 */ 7318 ioc->page_size = 1 << facts->CurrentHostPageSize; 7319 if (ioc->page_size == 1) { 7320 ioc_info(ioc, "CurrentHostPageSize is 0: Setting default host page size to 4k\n"); 7321 ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K; 7322 } 7323 dinitprintk(ioc, 7324 ioc_info(ioc, "CurrentHostPageSize(%d)\n", 7325 facts->CurrentHostPageSize)); 7326 7327 dinitprintk(ioc, 7328 ioc_info(ioc, "hba queue depth(%d), max chains per io(%d)\n", 7329 facts->RequestCredit, facts->MaxChainDepth)); 7330 dinitprintk(ioc, 7331 ioc_info(ioc, "request frame size(%d), reply frame size(%d)\n", 7332 facts->IOCRequestFrameSize * 4, 7333 facts->ReplyFrameSize * 4)); 7334 return 0; 7335 } 7336 7337 /** 7338 * _base_send_ioc_init - send ioc_init to firmware 7339 * @ioc: per adapter object 7340 * 7341 * Return: 0 for success, non-zero for failure. 7342 */ 7343 static int 7344 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc) 7345 { 7346 Mpi2IOCInitRequest_t mpi_request; 7347 Mpi2IOCInitReply_t mpi_reply; 7348 int i, r = 0; 7349 ktime_t current_time; 7350 u16 ioc_status; 7351 u32 reply_post_free_array_sz = 0; 7352 7353 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7354 7355 memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t)); 7356 mpi_request.Function = MPI2_FUNCTION_IOC_INIT; 7357 mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER; 7358 mpi_request.VF_ID = 0; /* TODO */ 7359 mpi_request.VP_ID = 0; 7360 mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged); 7361 mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION); 7362 mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K; 7363 7364 if (_base_is_controller_msix_enabled(ioc)) 7365 mpi_request.HostMSIxVectors = ioc->reply_queue_count; 7366 mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4); 7367 mpi_request.ReplyDescriptorPostQueueDepth = 7368 cpu_to_le16(ioc->reply_post_queue_depth); 7369 mpi_request.ReplyFreeQueueDepth = 7370 cpu_to_le16(ioc->reply_free_queue_depth); 7371 7372 mpi_request.SenseBufferAddressHigh = 7373 cpu_to_le32((u64)ioc->sense_dma >> 32); 7374 mpi_request.SystemReplyAddressHigh = 7375 cpu_to_le32((u64)ioc->reply_dma >> 32); 7376 mpi_request.SystemRequestFrameBaseAddress = 7377 cpu_to_le64((u64)ioc->request_dma); 7378 mpi_request.ReplyFreeQueueAddress = 7379 cpu_to_le64((u64)ioc->reply_free_dma); 7380 7381 if (ioc->rdpq_array_enable) { 7382 reply_post_free_array_sz = ioc->reply_queue_count * 7383 sizeof(Mpi2IOCInitRDPQArrayEntry); 7384 memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz); 7385 for (i = 0; i < ioc->reply_queue_count; i++) 7386 ioc->reply_post_free_array[i].RDPQBaseAddress = 7387 cpu_to_le64( 7388 (u64)ioc->reply_post[i].reply_post_free_dma); 7389 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE; 7390 mpi_request.ReplyDescriptorPostQueueAddress = 7391 cpu_to_le64((u64)ioc->reply_post_free_array_dma); 7392 } else { 7393 mpi_request.ReplyDescriptorPostQueueAddress = 7394 cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma); 7395 } 7396 7397 /* 7398 * Set the flag to enable CoreDump state feature in IOC firmware. 7399 */ 7400 mpi_request.ConfigurationFlags |= 7401 cpu_to_le16(MPI26_IOCINIT_CFGFLAGS_COREDUMP_ENABLE); 7402 7403 /* This time stamp specifies number of milliseconds 7404 * since epoch ~ midnight January 1, 1970. 7405 */ 7406 current_time = ktime_get_real(); 7407 mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time)); 7408 7409 if (ioc->logging_level & MPT_DEBUG_INIT) { 7410 __le32 *mfp; 7411 int i; 7412 7413 mfp = (__le32 *)&mpi_request; 7414 ioc_info(ioc, "\toffset:data\n"); 7415 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++) 7416 ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4, 7417 le32_to_cpu(mfp[i])); 7418 } 7419 7420 r = _base_handshake_req_reply_wait(ioc, 7421 sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request, 7422 sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 30); 7423 7424 if (r != 0) { 7425 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); 7426 return r; 7427 } 7428 7429 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK; 7430 if (ioc_status != MPI2_IOCSTATUS_SUCCESS || 7431 mpi_reply.IOCLogInfo) { 7432 ioc_err(ioc, "%s: failed\n", __func__); 7433 r = -EIO; 7434 } 7435 7436 /* Reset TimeSync Counter*/ 7437 ioc->timestamp_update_count = 0; 7438 return r; 7439 } 7440 7441 /** 7442 * mpt3sas_port_enable_done - command completion routine for port enable 7443 * @ioc: per adapter object 7444 * @smid: system request message index 7445 * @msix_index: MSIX table index supplied by the OS 7446 * @reply: reply message frame(lower 32bit addr) 7447 * 7448 * Return: 1 meaning mf should be freed from _base_interrupt 7449 * 0 means the mf is freed from this function. 7450 */ 7451 u8 7452 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, 7453 u32 reply) 7454 { 7455 MPI2DefaultReply_t *mpi_reply; 7456 u16 ioc_status; 7457 7458 if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED) 7459 return 1; 7460 7461 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 7462 if (!mpi_reply) 7463 return 1; 7464 7465 if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE) 7466 return 1; 7467 7468 ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING; 7469 ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE; 7470 ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID; 7471 memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4); 7472 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK; 7473 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) 7474 ioc->port_enable_failed = 1; 7475 7476 if (ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE_ASYNC) { 7477 ioc->port_enable_cmds.status &= ~MPT3_CMD_COMPLETE_ASYNC; 7478 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) { 7479 mpt3sas_port_enable_complete(ioc); 7480 return 1; 7481 } else { 7482 ioc->start_scan_failed = ioc_status; 7483 ioc->start_scan = 0; 7484 return 1; 7485 } 7486 } 7487 complete(&ioc->port_enable_cmds.done); 7488 return 1; 7489 } 7490 7491 /** 7492 * _base_send_port_enable - send port_enable(discovery stuff) to firmware 7493 * @ioc: per adapter object 7494 * 7495 * Return: 0 for success, non-zero for failure. 7496 */ 7497 static int 7498 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc) 7499 { 7500 Mpi2PortEnableRequest_t *mpi_request; 7501 Mpi2PortEnableReply_t *mpi_reply; 7502 int r = 0; 7503 u16 smid; 7504 u16 ioc_status; 7505 7506 ioc_info(ioc, "sending port enable !!\n"); 7507 7508 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) { 7509 ioc_err(ioc, "%s: internal command already in use\n", __func__); 7510 return -EAGAIN; 7511 } 7512 7513 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx); 7514 if (!smid) { 7515 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7516 return -EAGAIN; 7517 } 7518 7519 ioc->port_enable_cmds.status = MPT3_CMD_PENDING; 7520 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 7521 ioc->port_enable_cmds.smid = smid; 7522 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t)); 7523 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; 7524 7525 init_completion(&ioc->port_enable_cmds.done); 7526 ioc->put_smid_default(ioc, smid); 7527 wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ); 7528 if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) { 7529 ioc_err(ioc, "%s: timeout\n", __func__); 7530 _debug_dump_mf(mpi_request, 7531 sizeof(Mpi2PortEnableRequest_t)/4); 7532 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) 7533 r = -EFAULT; 7534 else 7535 r = -ETIME; 7536 goto out; 7537 } 7538 7539 mpi_reply = ioc->port_enable_cmds.reply; 7540 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK; 7541 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 7542 ioc_err(ioc, "%s: failed with (ioc_status=0x%08x)\n", 7543 __func__, ioc_status); 7544 r = -EFAULT; 7545 goto out; 7546 } 7547 7548 out: 7549 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED; 7550 ioc_info(ioc, "port enable: %s\n", r == 0 ? "SUCCESS" : "FAILED"); 7551 return r; 7552 } 7553 7554 /** 7555 * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply) 7556 * @ioc: per adapter object 7557 * 7558 * Return: 0 for success, non-zero for failure. 7559 */ 7560 int 7561 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc) 7562 { 7563 Mpi2PortEnableRequest_t *mpi_request; 7564 u16 smid; 7565 7566 ioc_info(ioc, "sending port enable !!\n"); 7567 7568 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) { 7569 ioc_err(ioc, "%s: internal command already in use\n", __func__); 7570 return -EAGAIN; 7571 } 7572 7573 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx); 7574 if (!smid) { 7575 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7576 return -EAGAIN; 7577 } 7578 ioc->drv_internal_flags |= MPT_DRV_INTERNAL_FIRST_PE_ISSUED; 7579 ioc->port_enable_cmds.status = MPT3_CMD_PENDING; 7580 ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE_ASYNC; 7581 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 7582 ioc->port_enable_cmds.smid = smid; 7583 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t)); 7584 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; 7585 7586 ioc->put_smid_default(ioc, smid); 7587 return 0; 7588 } 7589 7590 /** 7591 * _base_determine_wait_on_discovery - desposition 7592 * @ioc: per adapter object 7593 * 7594 * Decide whether to wait on discovery to complete. Used to either 7595 * locate boot device, or report volumes ahead of physical devices. 7596 * 7597 * Return: 1 for wait, 0 for don't wait. 7598 */ 7599 static int 7600 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc) 7601 { 7602 /* We wait for discovery to complete if IR firmware is loaded. 7603 * The sas topology events arrive before PD events, so we need time to 7604 * turn on the bit in ioc->pd_handles to indicate PD 7605 * Also, it maybe required to report Volumes ahead of physical 7606 * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set. 7607 */ 7608 if (ioc->ir_firmware) 7609 return 1; 7610 7611 /* if no Bios, then we don't need to wait */ 7612 if (!ioc->bios_pg3.BiosVersion) 7613 return 0; 7614 7615 /* Bios is present, then we drop down here. 7616 * 7617 * If there any entries in the Bios Page 2, then we wait 7618 * for discovery to complete. 7619 */ 7620 7621 /* Current Boot Device */ 7622 if ((ioc->bios_pg2.CurrentBootDeviceForm & 7623 MPI2_BIOSPAGE2_FORM_MASK) == 7624 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED && 7625 /* Request Boot Device */ 7626 (ioc->bios_pg2.ReqBootDeviceForm & 7627 MPI2_BIOSPAGE2_FORM_MASK) == 7628 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED && 7629 /* Alternate Request Boot Device */ 7630 (ioc->bios_pg2.ReqAltBootDeviceForm & 7631 MPI2_BIOSPAGE2_FORM_MASK) == 7632 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED) 7633 return 0; 7634 7635 return 1; 7636 } 7637 7638 /** 7639 * _base_unmask_events - turn on notification for this event 7640 * @ioc: per adapter object 7641 * @event: firmware event 7642 * 7643 * The mask is stored in ioc->event_masks. 7644 */ 7645 static void 7646 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event) 7647 { 7648 u32 desired_event; 7649 7650 if (event >= 128) 7651 return; 7652 7653 desired_event = (1 << (event % 32)); 7654 7655 if (event < 32) 7656 ioc->event_masks[0] &= ~desired_event; 7657 else if (event < 64) 7658 ioc->event_masks[1] &= ~desired_event; 7659 else if (event < 96) 7660 ioc->event_masks[2] &= ~desired_event; 7661 else if (event < 128) 7662 ioc->event_masks[3] &= ~desired_event; 7663 } 7664 7665 /** 7666 * _base_event_notification - send event notification 7667 * @ioc: per adapter object 7668 * 7669 * Return: 0 for success, non-zero for failure. 7670 */ 7671 static int 7672 _base_event_notification(struct MPT3SAS_ADAPTER *ioc) 7673 { 7674 Mpi2EventNotificationRequest_t *mpi_request; 7675 u16 smid; 7676 int r = 0; 7677 int i, issue_diag_reset = 0; 7678 7679 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7680 7681 if (ioc->base_cmds.status & MPT3_CMD_PENDING) { 7682 ioc_err(ioc, "%s: internal command already in use\n", __func__); 7683 return -EAGAIN; 7684 } 7685 7686 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 7687 if (!smid) { 7688 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7689 return -EAGAIN; 7690 } 7691 ioc->base_cmds.status = MPT3_CMD_PENDING; 7692 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 7693 ioc->base_cmds.smid = smid; 7694 memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t)); 7695 mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION; 7696 mpi_request->VF_ID = 0; /* TODO */ 7697 mpi_request->VP_ID = 0; 7698 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) 7699 mpi_request->EventMasks[i] = 7700 cpu_to_le32(ioc->event_masks[i]); 7701 init_completion(&ioc->base_cmds.done); 7702 ioc->put_smid_default(ioc, smid); 7703 wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ); 7704 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 7705 ioc_err(ioc, "%s: timeout\n", __func__); 7706 _debug_dump_mf(mpi_request, 7707 sizeof(Mpi2EventNotificationRequest_t)/4); 7708 if (ioc->base_cmds.status & MPT3_CMD_RESET) 7709 r = -EFAULT; 7710 else 7711 issue_diag_reset = 1; 7712 7713 } else 7714 dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__)); 7715 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7716 7717 if (issue_diag_reset) { 7718 if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED) 7719 return -EFAULT; 7720 if (mpt3sas_base_check_for_fault_and_issue_reset(ioc)) 7721 return -EFAULT; 7722 r = -EAGAIN; 7723 } 7724 return r; 7725 } 7726 7727 /** 7728 * mpt3sas_base_validate_event_type - validating event types 7729 * @ioc: per adapter object 7730 * @event_type: firmware event 7731 * 7732 * This will turn on firmware event notification when application 7733 * ask for that event. We don't mask events that are already enabled. 7734 */ 7735 void 7736 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type) 7737 { 7738 int i, j; 7739 u32 event_mask, desired_event; 7740 u8 send_update_to_fw; 7741 7742 for (i = 0, send_update_to_fw = 0; i < 7743 MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) { 7744 event_mask = ~event_type[i]; 7745 desired_event = 1; 7746 for (j = 0; j < 32; j++) { 7747 if (!(event_mask & desired_event) && 7748 (ioc->event_masks[i] & desired_event)) { 7749 ioc->event_masks[i] &= ~desired_event; 7750 send_update_to_fw = 1; 7751 } 7752 desired_event = (desired_event << 1); 7753 } 7754 } 7755 7756 if (!send_update_to_fw) 7757 return; 7758 7759 mutex_lock(&ioc->base_cmds.mutex); 7760 _base_event_notification(ioc); 7761 mutex_unlock(&ioc->base_cmds.mutex); 7762 } 7763 7764 /** 7765 * _base_diag_reset - the "big hammer" start of day reset 7766 * @ioc: per adapter object 7767 * 7768 * Return: 0 for success, non-zero for failure. 7769 */ 7770 static int 7771 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc) 7772 { 7773 u32 host_diagnostic; 7774 u32 ioc_state; 7775 u32 count; 7776 u32 hcb_size; 7777 7778 ioc_info(ioc, "sending diag reset !!\n"); 7779 7780 pci_cfg_access_lock(ioc->pdev); 7781 7782 drsprintk(ioc, ioc_info(ioc, "clear interrupts\n")); 7783 7784 count = 0; 7785 do { 7786 /* Write magic sequence to WriteSequence register 7787 * Loop until in diagnostic mode 7788 */ 7789 drsprintk(ioc, ioc_info(ioc, "write magic sequence\n")); 7790 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence); 7791 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence); 7792 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence); 7793 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence); 7794 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence); 7795 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence); 7796 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence); 7797 7798 /* wait 100 msec */ 7799 msleep(100); 7800 7801 if (count++ > 20) { 7802 ioc_info(ioc, 7803 "Stop writing magic sequence after 20 retries\n"); 7804 _base_dump_reg_set(ioc); 7805 goto out; 7806 } 7807 7808 host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic); 7809 drsprintk(ioc, 7810 ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n", 7811 count, host_diagnostic)); 7812 7813 } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0); 7814 7815 hcb_size = ioc->base_readl(&ioc->chip->HCBSize); 7816 7817 drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n")); 7818 writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER, 7819 &ioc->chip->HostDiagnostic); 7820 7821 /*This delay allows the chip PCIe hardware time to finish reset tasks*/ 7822 msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000); 7823 7824 /* Approximately 300 second max wait */ 7825 for (count = 0; count < (300000000 / 7826 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) { 7827 7828 host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic); 7829 7830 if (host_diagnostic == 0xFFFFFFFF) { 7831 ioc_info(ioc, 7832 "Invalid host diagnostic register value\n"); 7833 _base_dump_reg_set(ioc); 7834 goto out; 7835 } 7836 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER)) 7837 break; 7838 7839 msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000); 7840 } 7841 7842 if (host_diagnostic & MPI2_DIAG_HCB_MODE) { 7843 7844 drsprintk(ioc, 7845 ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n")); 7846 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK; 7847 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW; 7848 writel(host_diagnostic, &ioc->chip->HostDiagnostic); 7849 7850 drsprintk(ioc, ioc_info(ioc, "re-enable the HCDW\n")); 7851 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE, 7852 &ioc->chip->HCBSize); 7853 } 7854 7855 drsprintk(ioc, ioc_info(ioc, "restart the adapter\n")); 7856 writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET, 7857 &ioc->chip->HostDiagnostic); 7858 7859 drsprintk(ioc, 7860 ioc_info(ioc, "disable writes to the diagnostic register\n")); 7861 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence); 7862 7863 drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n")); 7864 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20); 7865 if (ioc_state) { 7866 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 7867 __func__, ioc_state); 7868 _base_dump_reg_set(ioc); 7869 goto out; 7870 } 7871 7872 pci_cfg_access_unlock(ioc->pdev); 7873 ioc_info(ioc, "diag reset: SUCCESS\n"); 7874 return 0; 7875 7876 out: 7877 pci_cfg_access_unlock(ioc->pdev); 7878 ioc_err(ioc, "diag reset: FAILED\n"); 7879 return -EFAULT; 7880 } 7881 7882 /** 7883 * mpt3sas_base_make_ioc_ready - put controller in READY state 7884 * @ioc: per adapter object 7885 * @type: FORCE_BIG_HAMMER or SOFT_RESET 7886 * 7887 * Return: 0 for success, non-zero for failure. 7888 */ 7889 int 7890 mpt3sas_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type) 7891 { 7892 u32 ioc_state; 7893 int rc; 7894 int count; 7895 7896 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7897 7898 if (ioc->pci_error_recovery) 7899 return 0; 7900 7901 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 7902 dhsprintk(ioc, 7903 ioc_info(ioc, "%s: ioc_state(0x%08x)\n", 7904 __func__, ioc_state)); 7905 7906 /* if in RESET state, it should move to READY state shortly */ 7907 count = 0; 7908 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) { 7909 while ((ioc_state & MPI2_IOC_STATE_MASK) != 7910 MPI2_IOC_STATE_READY) { 7911 if (count++ == 10) { 7912 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 7913 __func__, ioc_state); 7914 return -EFAULT; 7915 } 7916 ssleep(1); 7917 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 7918 } 7919 } 7920 7921 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) 7922 return 0; 7923 7924 if (ioc_state & MPI2_DOORBELL_USED) { 7925 ioc_info(ioc, "unexpected doorbell active!\n"); 7926 goto issue_diag_reset; 7927 } 7928 7929 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 7930 mpt3sas_print_fault_code(ioc, ioc_state & 7931 MPI2_DOORBELL_DATA_MASK); 7932 goto issue_diag_reset; 7933 } 7934 7935 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) { 7936 /* 7937 * if host reset is invoked while watch dog thread is waiting 7938 * for IOC state to be changed to Fault state then driver has 7939 * to wait here for CoreDump state to clear otherwise reset 7940 * will be issued to the FW and FW move the IOC state to 7941 * reset state without copying the FW logs to coredump region. 7942 */ 7943 if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) { 7944 mpt3sas_print_coredump_info(ioc, ioc_state & 7945 MPI2_DOORBELL_DATA_MASK); 7946 mpt3sas_base_wait_for_coredump_completion(ioc, 7947 __func__); 7948 } 7949 goto issue_diag_reset; 7950 } 7951 7952 if (type == FORCE_BIG_HAMMER) 7953 goto issue_diag_reset; 7954 7955 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL) 7956 if (!(_base_send_ioc_reset(ioc, 7957 MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) { 7958 return 0; 7959 } 7960 7961 issue_diag_reset: 7962 rc = _base_diag_reset(ioc); 7963 return rc; 7964 } 7965 7966 /** 7967 * _base_make_ioc_operational - put controller in OPERATIONAL state 7968 * @ioc: per adapter object 7969 * 7970 * Return: 0 for success, non-zero for failure. 7971 */ 7972 static int 7973 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) 7974 { 7975 int r, i, index, rc; 7976 unsigned long flags; 7977 u32 reply_address; 7978 u16 smid; 7979 struct _tr_list *delayed_tr, *delayed_tr_next; 7980 struct _sc_list *delayed_sc, *delayed_sc_next; 7981 struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next; 7982 u8 hide_flag; 7983 struct adapter_reply_queue *reply_q; 7984 Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig; 7985 7986 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7987 7988 /* clean the delayed target reset list */ 7989 list_for_each_entry_safe(delayed_tr, delayed_tr_next, 7990 &ioc->delayed_tr_list, list) { 7991 list_del(&delayed_tr->list); 7992 kfree(delayed_tr); 7993 } 7994 7995 7996 list_for_each_entry_safe(delayed_tr, delayed_tr_next, 7997 &ioc->delayed_tr_volume_list, list) { 7998 list_del(&delayed_tr->list); 7999 kfree(delayed_tr); 8000 } 8001 8002 list_for_each_entry_safe(delayed_sc, delayed_sc_next, 8003 &ioc->delayed_sc_list, list) { 8004 list_del(&delayed_sc->list); 8005 kfree(delayed_sc); 8006 } 8007 8008 list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next, 8009 &ioc->delayed_event_ack_list, list) { 8010 list_del(&delayed_event_ack->list); 8011 kfree(delayed_event_ack); 8012 } 8013 8014 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 8015 8016 /* hi-priority queue */ 8017 INIT_LIST_HEAD(&ioc->hpr_free_list); 8018 smid = ioc->hi_priority_smid; 8019 for (i = 0; i < ioc->hi_priority_depth; i++, smid++) { 8020 ioc->hpr_lookup[i].cb_idx = 0xFF; 8021 ioc->hpr_lookup[i].smid = smid; 8022 list_add_tail(&ioc->hpr_lookup[i].tracker_list, 8023 &ioc->hpr_free_list); 8024 } 8025 8026 /* internal queue */ 8027 INIT_LIST_HEAD(&ioc->internal_free_list); 8028 smid = ioc->internal_smid; 8029 for (i = 0; i < ioc->internal_depth; i++, smid++) { 8030 ioc->internal_lookup[i].cb_idx = 0xFF; 8031 ioc->internal_lookup[i].smid = smid; 8032 list_add_tail(&ioc->internal_lookup[i].tracker_list, 8033 &ioc->internal_free_list); 8034 } 8035 8036 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 8037 8038 /* initialize Reply Free Queue */ 8039 for (i = 0, reply_address = (u32)ioc->reply_dma ; 8040 i < ioc->reply_free_queue_depth ; i++, reply_address += 8041 ioc->reply_sz) { 8042 ioc->reply_free[i] = cpu_to_le32(reply_address); 8043 if (ioc->is_mcpu_endpoint) 8044 _base_clone_reply_to_sys_mem(ioc, 8045 reply_address, i); 8046 } 8047 8048 /* initialize reply queues */ 8049 if (ioc->is_driver_loading) 8050 _base_assign_reply_queues(ioc); 8051 8052 /* initialize Reply Post Free Queue */ 8053 index = 0; 8054 reply_post_free_contig = ioc->reply_post[0].reply_post_free; 8055 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 8056 /* 8057 * If RDPQ is enabled, switch to the next allocation. 8058 * Otherwise advance within the contiguous region. 8059 */ 8060 if (ioc->rdpq_array_enable) { 8061 reply_q->reply_post_free = 8062 ioc->reply_post[index++].reply_post_free; 8063 } else { 8064 reply_q->reply_post_free = reply_post_free_contig; 8065 reply_post_free_contig += ioc->reply_post_queue_depth; 8066 } 8067 8068 reply_q->reply_post_host_index = 0; 8069 for (i = 0; i < ioc->reply_post_queue_depth; i++) 8070 reply_q->reply_post_free[i].Words = 8071 cpu_to_le64(ULLONG_MAX); 8072 if (!_base_is_controller_msix_enabled(ioc)) 8073 goto skip_init_reply_post_free_queue; 8074 } 8075 skip_init_reply_post_free_queue: 8076 8077 r = _base_send_ioc_init(ioc); 8078 if (r) { 8079 /* 8080 * No need to check IOC state for fault state & issue 8081 * diag reset during host reset. This check is need 8082 * only during driver load time. 8083 */ 8084 if (!ioc->is_driver_loading) 8085 return r; 8086 8087 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 8088 if (rc || (_base_send_ioc_init(ioc))) 8089 return r; 8090 } 8091 8092 /* initialize reply free host index */ 8093 ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1; 8094 writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex); 8095 8096 /* initialize reply post host index */ 8097 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 8098 if (ioc->combined_reply_queue) 8099 writel((reply_q->msix_index & 7)<< 8100 MPI2_RPHI_MSIX_INDEX_SHIFT, 8101 ioc->replyPostRegisterIndex[reply_q->msix_index/8]); 8102 else 8103 writel(reply_q->msix_index << 8104 MPI2_RPHI_MSIX_INDEX_SHIFT, 8105 &ioc->chip->ReplyPostHostIndex); 8106 8107 if (!_base_is_controller_msix_enabled(ioc)) 8108 goto skip_init_reply_post_host_index; 8109 } 8110 8111 skip_init_reply_post_host_index: 8112 8113 mpt3sas_base_unmask_interrupts(ioc); 8114 8115 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { 8116 r = _base_display_fwpkg_version(ioc); 8117 if (r) 8118 return r; 8119 } 8120 8121 r = _base_static_config_pages(ioc); 8122 if (r) 8123 return r; 8124 8125 r = _base_event_notification(ioc); 8126 if (r) 8127 return r; 8128 8129 if (!ioc->shost_recovery) { 8130 8131 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier 8132 == 0x80) { 8133 hide_flag = (u8) ( 8134 le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) & 8135 MFG_PAGE10_HIDE_SSDS_MASK); 8136 if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK) 8137 ioc->mfg_pg10_hide_flag = hide_flag; 8138 } 8139 8140 ioc->wait_for_discovery_to_complete = 8141 _base_determine_wait_on_discovery(ioc); 8142 8143 return r; /* scan_start and scan_finished support */ 8144 } 8145 8146 r = _base_send_port_enable(ioc); 8147 if (r) 8148 return r; 8149 8150 return r; 8151 } 8152 8153 /** 8154 * mpt3sas_base_free_resources - free resources controller resources 8155 * @ioc: per adapter object 8156 */ 8157 void 8158 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc) 8159 { 8160 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 8161 8162 /* synchronizing freeing resource with pci_access_mutex lock */ 8163 mutex_lock(&ioc->pci_access_mutex); 8164 if (ioc->chip_phys && ioc->chip) { 8165 mpt3sas_base_mask_interrupts(ioc); 8166 ioc->shost_recovery = 1; 8167 mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET); 8168 ioc->shost_recovery = 0; 8169 } 8170 8171 mpt3sas_base_unmap_resources(ioc); 8172 mutex_unlock(&ioc->pci_access_mutex); 8173 return; 8174 } 8175 8176 /** 8177 * mpt3sas_base_attach - attach controller instance 8178 * @ioc: per adapter object 8179 * 8180 * Return: 0 for success, non-zero for failure. 8181 */ 8182 int 8183 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) 8184 { 8185 int r, i, rc; 8186 int cpu_id, last_cpu_id = 0; 8187 8188 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 8189 8190 /* setup cpu_msix_table */ 8191 ioc->cpu_count = num_online_cpus(); 8192 for_each_online_cpu(cpu_id) 8193 last_cpu_id = cpu_id; 8194 ioc->cpu_msix_table_sz = last_cpu_id + 1; 8195 ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL); 8196 ioc->reply_queue_count = 1; 8197 if (!ioc->cpu_msix_table) { 8198 ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n"); 8199 r = -ENOMEM; 8200 goto out_free_resources; 8201 } 8202 8203 if (ioc->is_warpdrive) { 8204 ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz, 8205 sizeof(resource_size_t *), GFP_KERNEL); 8206 if (!ioc->reply_post_host_index) { 8207 ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n"); 8208 r = -ENOMEM; 8209 goto out_free_resources; 8210 } 8211 } 8212 8213 ioc->smp_affinity_enable = smp_affinity_enable; 8214 8215 ioc->rdpq_array_enable_assigned = 0; 8216 ioc->use_32bit_dma = false; 8217 ioc->dma_mask = 64; 8218 if (ioc->is_aero_ioc) 8219 ioc->base_readl = &_base_readl_aero; 8220 else 8221 ioc->base_readl = &_base_readl; 8222 r = mpt3sas_base_map_resources(ioc); 8223 if (r) 8224 goto out_free_resources; 8225 8226 pci_set_drvdata(ioc->pdev, ioc->shost); 8227 r = _base_get_ioc_facts(ioc); 8228 if (r) { 8229 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 8230 if (rc || (_base_get_ioc_facts(ioc))) 8231 goto out_free_resources; 8232 } 8233 8234 switch (ioc->hba_mpi_version_belonged) { 8235 case MPI2_VERSION: 8236 ioc->build_sg_scmd = &_base_build_sg_scmd; 8237 ioc->build_sg = &_base_build_sg; 8238 ioc->build_zero_len_sge = &_base_build_zero_len_sge; 8239 ioc->get_msix_index_for_smlio = &_base_get_msix_index; 8240 break; 8241 case MPI25_VERSION: 8242 case MPI26_VERSION: 8243 /* 8244 * In SAS3.0, 8245 * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and 8246 * Target Status - all require the IEEE formatted scatter gather 8247 * elements. 8248 */ 8249 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee; 8250 ioc->build_sg = &_base_build_sg_ieee; 8251 ioc->build_nvme_prp = &_base_build_nvme_prp; 8252 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee; 8253 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t); 8254 if (ioc->high_iops_queues) 8255 ioc->get_msix_index_for_smlio = 8256 &_base_get_high_iops_msix_index; 8257 else 8258 ioc->get_msix_index_for_smlio = &_base_get_msix_index; 8259 break; 8260 } 8261 if (ioc->atomic_desc_capable) { 8262 ioc->put_smid_default = &_base_put_smid_default_atomic; 8263 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic; 8264 ioc->put_smid_fast_path = 8265 &_base_put_smid_fast_path_atomic; 8266 ioc->put_smid_hi_priority = 8267 &_base_put_smid_hi_priority_atomic; 8268 } else { 8269 ioc->put_smid_default = &_base_put_smid_default; 8270 ioc->put_smid_fast_path = &_base_put_smid_fast_path; 8271 ioc->put_smid_hi_priority = &_base_put_smid_hi_priority; 8272 if (ioc->is_mcpu_endpoint) 8273 ioc->put_smid_scsi_io = 8274 &_base_put_smid_mpi_ep_scsi_io; 8275 else 8276 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io; 8277 } 8278 /* 8279 * These function pointers for other requests that don't 8280 * the require IEEE scatter gather elements. 8281 * 8282 * For example Configuration Pages and SAS IOUNIT Control don't. 8283 */ 8284 ioc->build_sg_mpi = &_base_build_sg; 8285 ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge; 8286 8287 r = mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET); 8288 if (r) 8289 goto out_free_resources; 8290 8291 ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts, 8292 sizeof(struct mpt3sas_port_facts), GFP_KERNEL); 8293 if (!ioc->pfacts) { 8294 r = -ENOMEM; 8295 goto out_free_resources; 8296 } 8297 8298 for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) { 8299 r = _base_get_port_facts(ioc, i); 8300 if (r) { 8301 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 8302 if (rc || (_base_get_port_facts(ioc, i))) 8303 goto out_free_resources; 8304 } 8305 } 8306 8307 r = _base_allocate_memory_pools(ioc); 8308 if (r) 8309 goto out_free_resources; 8310 8311 if (irqpoll_weight > 0) 8312 ioc->thresh_hold = irqpoll_weight; 8313 else 8314 ioc->thresh_hold = ioc->hba_queue_depth/4; 8315 8316 _base_init_irqpolls(ioc); 8317 init_waitqueue_head(&ioc->reset_wq); 8318 8319 /* allocate memory pd handle bitmask list */ 8320 ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8); 8321 if (ioc->facts.MaxDevHandle % 8) 8322 ioc->pd_handles_sz++; 8323 ioc->pd_handles = kzalloc(ioc->pd_handles_sz, 8324 GFP_KERNEL); 8325 if (!ioc->pd_handles) { 8326 r = -ENOMEM; 8327 goto out_free_resources; 8328 } 8329 ioc->blocking_handles = kzalloc(ioc->pd_handles_sz, 8330 GFP_KERNEL); 8331 if (!ioc->blocking_handles) { 8332 r = -ENOMEM; 8333 goto out_free_resources; 8334 } 8335 8336 /* allocate memory for pending OS device add list */ 8337 ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8); 8338 if (ioc->facts.MaxDevHandle % 8) 8339 ioc->pend_os_device_add_sz++; 8340 ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz, 8341 GFP_KERNEL); 8342 if (!ioc->pend_os_device_add) { 8343 r = -ENOMEM; 8344 goto out_free_resources; 8345 } 8346 8347 ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz; 8348 ioc->device_remove_in_progress = 8349 kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL); 8350 if (!ioc->device_remove_in_progress) { 8351 r = -ENOMEM; 8352 goto out_free_resources; 8353 } 8354 8355 ioc->fwfault_debug = mpt3sas_fwfault_debug; 8356 8357 /* base internal command bits */ 8358 mutex_init(&ioc->base_cmds.mutex); 8359 ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8360 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 8361 8362 /* port_enable command bits */ 8363 ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8364 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED; 8365 8366 /* transport internal command bits */ 8367 ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8368 ioc->transport_cmds.status = MPT3_CMD_NOT_USED; 8369 mutex_init(&ioc->transport_cmds.mutex); 8370 8371 /* scsih internal command bits */ 8372 ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8373 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; 8374 mutex_init(&ioc->scsih_cmds.mutex); 8375 8376 /* task management internal command bits */ 8377 ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8378 ioc->tm_cmds.status = MPT3_CMD_NOT_USED; 8379 mutex_init(&ioc->tm_cmds.mutex); 8380 8381 /* config page internal command bits */ 8382 ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8383 ioc->config_cmds.status = MPT3_CMD_NOT_USED; 8384 mutex_init(&ioc->config_cmds.mutex); 8385 8386 /* ctl module internal command bits */ 8387 ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8388 ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); 8389 ioc->ctl_cmds.status = MPT3_CMD_NOT_USED; 8390 mutex_init(&ioc->ctl_cmds.mutex); 8391 8392 if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply || 8393 !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply || 8394 !ioc->tm_cmds.reply || !ioc->config_cmds.reply || 8395 !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) { 8396 r = -ENOMEM; 8397 goto out_free_resources; 8398 } 8399 8400 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) 8401 ioc->event_masks[i] = -1; 8402 8403 /* here we enable the events we care about */ 8404 _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY); 8405 _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE); 8406 _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST); 8407 _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE); 8408 _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE); 8409 _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST); 8410 _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME); 8411 _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK); 8412 _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS); 8413 _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED); 8414 _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD); 8415 _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION); 8416 _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR); 8417 if (ioc->hba_mpi_version_belonged == MPI26_VERSION) { 8418 if (ioc->is_gen35_ioc) { 8419 _base_unmask_events(ioc, 8420 MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE); 8421 _base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION); 8422 _base_unmask_events(ioc, 8423 MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST); 8424 } 8425 } 8426 r = _base_make_ioc_operational(ioc); 8427 if (r == -EAGAIN) { 8428 r = _base_make_ioc_operational(ioc); 8429 if (r) 8430 goto out_free_resources; 8431 } 8432 8433 /* 8434 * Copy current copy of IOCFacts in prev_fw_facts 8435 * and it will be used during online firmware upgrade. 8436 */ 8437 memcpy(&ioc->prev_fw_facts, &ioc->facts, 8438 sizeof(struct mpt3sas_facts)); 8439 8440 ioc->non_operational_loop = 0; 8441 ioc->ioc_coredump_loop = 0; 8442 ioc->got_task_abort_from_ioctl = 0; 8443 return 0; 8444 8445 out_free_resources: 8446 8447 ioc->remove_host = 1; 8448 8449 mpt3sas_base_free_resources(ioc); 8450 _base_release_memory_pools(ioc); 8451 pci_set_drvdata(ioc->pdev, NULL); 8452 kfree(ioc->cpu_msix_table); 8453 if (ioc->is_warpdrive) 8454 kfree(ioc->reply_post_host_index); 8455 kfree(ioc->pd_handles); 8456 kfree(ioc->blocking_handles); 8457 kfree(ioc->device_remove_in_progress); 8458 kfree(ioc->pend_os_device_add); 8459 kfree(ioc->tm_cmds.reply); 8460 kfree(ioc->transport_cmds.reply); 8461 kfree(ioc->scsih_cmds.reply); 8462 kfree(ioc->config_cmds.reply); 8463 kfree(ioc->base_cmds.reply); 8464 kfree(ioc->port_enable_cmds.reply); 8465 kfree(ioc->ctl_cmds.reply); 8466 kfree(ioc->ctl_cmds.sense); 8467 kfree(ioc->pfacts); 8468 ioc->ctl_cmds.reply = NULL; 8469 ioc->base_cmds.reply = NULL; 8470 ioc->tm_cmds.reply = NULL; 8471 ioc->scsih_cmds.reply = NULL; 8472 ioc->transport_cmds.reply = NULL; 8473 ioc->config_cmds.reply = NULL; 8474 ioc->pfacts = NULL; 8475 return r; 8476 } 8477 8478 8479 /** 8480 * mpt3sas_base_detach - remove controller instance 8481 * @ioc: per adapter object 8482 */ 8483 void 8484 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc) 8485 { 8486 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 8487 8488 mpt3sas_base_stop_watchdog(ioc); 8489 mpt3sas_base_free_resources(ioc); 8490 _base_release_memory_pools(ioc); 8491 mpt3sas_free_enclosure_list(ioc); 8492 pci_set_drvdata(ioc->pdev, NULL); 8493 kfree(ioc->cpu_msix_table); 8494 if (ioc->is_warpdrive) 8495 kfree(ioc->reply_post_host_index); 8496 kfree(ioc->pd_handles); 8497 kfree(ioc->blocking_handles); 8498 kfree(ioc->device_remove_in_progress); 8499 kfree(ioc->pend_os_device_add); 8500 kfree(ioc->pfacts); 8501 kfree(ioc->ctl_cmds.reply); 8502 kfree(ioc->ctl_cmds.sense); 8503 kfree(ioc->base_cmds.reply); 8504 kfree(ioc->port_enable_cmds.reply); 8505 kfree(ioc->tm_cmds.reply); 8506 kfree(ioc->transport_cmds.reply); 8507 kfree(ioc->scsih_cmds.reply); 8508 kfree(ioc->config_cmds.reply); 8509 } 8510 8511 /** 8512 * _base_pre_reset_handler - pre reset handler 8513 * @ioc: per adapter object 8514 */ 8515 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc) 8516 { 8517 mpt3sas_scsih_pre_reset_handler(ioc); 8518 mpt3sas_ctl_pre_reset_handler(ioc); 8519 dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__)); 8520 } 8521 8522 /** 8523 * _base_clear_outstanding_mpt_commands - clears outstanding mpt commands 8524 * @ioc: per adapter object 8525 */ 8526 static void 8527 _base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc) 8528 { 8529 dtmprintk(ioc, 8530 ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__)); 8531 if (ioc->transport_cmds.status & MPT3_CMD_PENDING) { 8532 ioc->transport_cmds.status |= MPT3_CMD_RESET; 8533 mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid); 8534 complete(&ioc->transport_cmds.done); 8535 } 8536 if (ioc->base_cmds.status & MPT3_CMD_PENDING) { 8537 ioc->base_cmds.status |= MPT3_CMD_RESET; 8538 mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid); 8539 complete(&ioc->base_cmds.done); 8540 } 8541 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) { 8542 ioc->port_enable_failed = 1; 8543 ioc->port_enable_cmds.status |= MPT3_CMD_RESET; 8544 mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid); 8545 if (ioc->is_driver_loading) { 8546 ioc->start_scan_failed = 8547 MPI2_IOCSTATUS_INTERNAL_ERROR; 8548 ioc->start_scan = 0; 8549 } else { 8550 complete(&ioc->port_enable_cmds.done); 8551 } 8552 } 8553 if (ioc->config_cmds.status & MPT3_CMD_PENDING) { 8554 ioc->config_cmds.status |= MPT3_CMD_RESET; 8555 mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid); 8556 ioc->config_cmds.smid = USHRT_MAX; 8557 complete(&ioc->config_cmds.done); 8558 } 8559 } 8560 8561 /** 8562 * _base_clear_outstanding_commands - clear all outstanding commands 8563 * @ioc: per adapter object 8564 */ 8565 static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc) 8566 { 8567 mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc); 8568 mpt3sas_ctl_clear_outstanding_ioctls(ioc); 8569 _base_clear_outstanding_mpt_commands(ioc); 8570 } 8571 8572 /** 8573 * _base_reset_done_handler - reset done handler 8574 * @ioc: per adapter object 8575 */ 8576 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc) 8577 { 8578 mpt3sas_scsih_reset_done_handler(ioc); 8579 mpt3sas_ctl_reset_done_handler(ioc); 8580 dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__)); 8581 } 8582 8583 /** 8584 * mpt3sas_wait_for_commands_to_complete - reset controller 8585 * @ioc: Pointer to MPT_ADAPTER structure 8586 * 8587 * This function is waiting 10s for all pending commands to complete 8588 * prior to putting controller in reset. 8589 */ 8590 void 8591 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc) 8592 { 8593 u32 ioc_state; 8594 8595 ioc->pending_io_count = 0; 8596 8597 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 8598 if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) 8599 return; 8600 8601 /* pending command count */ 8602 ioc->pending_io_count = scsi_host_busy(ioc->shost); 8603 8604 if (!ioc->pending_io_count) 8605 return; 8606 8607 /* wait for pending commands to complete */ 8608 wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ); 8609 } 8610 8611 /** 8612 * _base_check_ioc_facts_changes - Look for increase/decrease of IOCFacts 8613 * attributes during online firmware upgrade and update the corresponding 8614 * IOC variables accordingly. 8615 * 8616 * @ioc: Pointer to MPT_ADAPTER structure 8617 */ 8618 static int 8619 _base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc) 8620 { 8621 u16 pd_handles_sz; 8622 void *pd_handles = NULL, *blocking_handles = NULL; 8623 void *pend_os_device_add = NULL, *device_remove_in_progress = NULL; 8624 struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts; 8625 8626 if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) { 8627 pd_handles_sz = (ioc->facts.MaxDevHandle / 8); 8628 if (ioc->facts.MaxDevHandle % 8) 8629 pd_handles_sz++; 8630 8631 pd_handles = krealloc(ioc->pd_handles, pd_handles_sz, 8632 GFP_KERNEL); 8633 if (!pd_handles) { 8634 ioc_info(ioc, 8635 "Unable to allocate the memory for pd_handles of sz: %d\n", 8636 pd_handles_sz); 8637 return -ENOMEM; 8638 } 8639 memset(pd_handles + ioc->pd_handles_sz, 0, 8640 (pd_handles_sz - ioc->pd_handles_sz)); 8641 ioc->pd_handles = pd_handles; 8642 8643 blocking_handles = krealloc(ioc->blocking_handles, 8644 pd_handles_sz, GFP_KERNEL); 8645 if (!blocking_handles) { 8646 ioc_info(ioc, 8647 "Unable to allocate the memory for " 8648 "blocking_handles of sz: %d\n", 8649 pd_handles_sz); 8650 return -ENOMEM; 8651 } 8652 memset(blocking_handles + ioc->pd_handles_sz, 0, 8653 (pd_handles_sz - ioc->pd_handles_sz)); 8654 ioc->blocking_handles = blocking_handles; 8655 ioc->pd_handles_sz = pd_handles_sz; 8656 8657 pend_os_device_add = krealloc(ioc->pend_os_device_add, 8658 pd_handles_sz, GFP_KERNEL); 8659 if (!pend_os_device_add) { 8660 ioc_info(ioc, 8661 "Unable to allocate the memory for pend_os_device_add of sz: %d\n", 8662 pd_handles_sz); 8663 return -ENOMEM; 8664 } 8665 memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0, 8666 (pd_handles_sz - ioc->pend_os_device_add_sz)); 8667 ioc->pend_os_device_add = pend_os_device_add; 8668 ioc->pend_os_device_add_sz = pd_handles_sz; 8669 8670 device_remove_in_progress = krealloc( 8671 ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL); 8672 if (!device_remove_in_progress) { 8673 ioc_info(ioc, 8674 "Unable to allocate the memory for " 8675 "device_remove_in_progress of sz: %d\n " 8676 , pd_handles_sz); 8677 return -ENOMEM; 8678 } 8679 memset(device_remove_in_progress + 8680 ioc->device_remove_in_progress_sz, 0, 8681 (pd_handles_sz - ioc->device_remove_in_progress_sz)); 8682 ioc->device_remove_in_progress = device_remove_in_progress; 8683 ioc->device_remove_in_progress_sz = pd_handles_sz; 8684 } 8685 8686 memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts)); 8687 return 0; 8688 } 8689 8690 /** 8691 * mpt3sas_base_hard_reset_handler - reset controller 8692 * @ioc: Pointer to MPT_ADAPTER structure 8693 * @type: FORCE_BIG_HAMMER or SOFT_RESET 8694 * 8695 * Return: 0 for success, non-zero for failure. 8696 */ 8697 int 8698 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, 8699 enum reset_type type) 8700 { 8701 int r; 8702 unsigned long flags; 8703 u32 ioc_state; 8704 u8 is_fault = 0, is_trigger = 0; 8705 8706 dtmprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__)); 8707 8708 if (ioc->pci_error_recovery) { 8709 ioc_err(ioc, "%s: pci error recovery reset\n", __func__); 8710 r = 0; 8711 goto out_unlocked; 8712 } 8713 8714 if (mpt3sas_fwfault_debug) 8715 mpt3sas_halt_firmware(ioc); 8716 8717 /* wait for an active reset in progress to complete */ 8718 mutex_lock(&ioc->reset_in_progress_mutex); 8719 8720 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 8721 ioc->shost_recovery = 1; 8722 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 8723 8724 if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] & 8725 MPT3_DIAG_BUFFER_IS_REGISTERED) && 8726 (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] & 8727 MPT3_DIAG_BUFFER_IS_RELEASED))) { 8728 is_trigger = 1; 8729 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 8730 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT || 8731 (ioc_state & MPI2_IOC_STATE_MASK) == 8732 MPI2_IOC_STATE_COREDUMP) { 8733 is_fault = 1; 8734 ioc->htb_rel.trigger_info_dwords[1] = 8735 (ioc_state & MPI2_DOORBELL_DATA_MASK); 8736 } 8737 } 8738 _base_pre_reset_handler(ioc); 8739 mpt3sas_wait_for_commands_to_complete(ioc); 8740 mpt3sas_base_mask_interrupts(ioc); 8741 mpt3sas_base_pause_mq_polling(ioc); 8742 r = mpt3sas_base_make_ioc_ready(ioc, type); 8743 if (r) 8744 goto out; 8745 _base_clear_outstanding_commands(ioc); 8746 8747 /* If this hard reset is called while port enable is active, then 8748 * there is no reason to call make_ioc_operational 8749 */ 8750 if (ioc->is_driver_loading && ioc->port_enable_failed) { 8751 ioc->remove_host = 1; 8752 r = -EFAULT; 8753 goto out; 8754 } 8755 r = _base_get_ioc_facts(ioc); 8756 if (r) 8757 goto out; 8758 8759 r = _base_check_ioc_facts_changes(ioc); 8760 if (r) { 8761 ioc_info(ioc, 8762 "Some of the parameters got changed in this new firmware" 8763 " image and it requires system reboot\n"); 8764 goto out; 8765 } 8766 if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable) 8767 panic("%s: Issue occurred with flashing controller firmware." 8768 "Please reboot the system and ensure that the correct" 8769 " firmware version is running\n", ioc->name); 8770 8771 r = _base_make_ioc_operational(ioc); 8772 if (!r) 8773 _base_reset_done_handler(ioc); 8774 8775 out: 8776 ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED"); 8777 8778 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 8779 ioc->shost_recovery = 0; 8780 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 8781 ioc->ioc_reset_count++; 8782 mutex_unlock(&ioc->reset_in_progress_mutex); 8783 mpt3sas_base_resume_mq_polling(ioc); 8784 8785 out_unlocked: 8786 if ((r == 0) && is_trigger) { 8787 if (is_fault) 8788 mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT); 8789 else 8790 mpt3sas_trigger_master(ioc, 8791 MASTER_TRIGGER_ADAPTER_RESET); 8792 } 8793 dtmprintk(ioc, ioc_info(ioc, "%s: exit\n", __func__)); 8794 return r; 8795 } 8796