1 /* 2 * Copyright(c) 2015 - 2017 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/pci.h> 49 #include <linux/io.h> 50 #include <linux/delay.h> 51 #include <linux/vmalloc.h> 52 #include <linux/aer.h> 53 #include <linux/module.h> 54 55 #include "hfi.h" 56 #include "chip_registers.h" 57 #include "aspm.h" 58 59 /* 60 * This file contains PCIe utility routines. 61 */ 62 63 /* 64 * Code to adjust PCIe capabilities. 65 */ 66 static void tune_pcie_caps(struct hfi1_devdata *); 67 68 /* 69 * Do all the common PCIe setup and initialization. 70 * devdata is not yet allocated, and is not allocated until after this 71 * routine returns success. Therefore dd_dev_err() can't be used for error 72 * printing. 73 */ 74 int hfi1_pcie_init(struct pci_dev *pdev, const struct pci_device_id *ent) 75 { 76 int ret; 77 78 ret = pci_enable_device(pdev); 79 if (ret) { 80 /* 81 * This can happen (in theory) iff: 82 * We did a chip reset, and then failed to reprogram the 83 * BAR, or the chip reset due to an internal error. We then 84 * unloaded the driver and reloaded it. 85 * 86 * Both reset cases set the BAR back to initial state. For 87 * the latter case, the AER sticky error bit at offset 0x718 88 * should be set, but the Linux kernel doesn't yet know 89 * about that, it appears. If the original BAR was retained 90 * in the kernel data structures, this may be OK. 91 */ 92 hfi1_early_err(&pdev->dev, "pci enable failed: error %d\n", 93 -ret); 94 goto done; 95 } 96 97 ret = pci_request_regions(pdev, DRIVER_NAME); 98 if (ret) { 99 hfi1_early_err(&pdev->dev, 100 "pci_request_regions fails: err %d\n", -ret); 101 goto bail; 102 } 103 104 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 105 if (ret) { 106 /* 107 * If the 64 bit setup fails, try 32 bit. Some systems 108 * do not setup 64 bit maps on systems with 2GB or less 109 * memory installed. 110 */ 111 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 112 if (ret) { 113 hfi1_early_err(&pdev->dev, 114 "Unable to set DMA mask: %d\n", ret); 115 goto bail; 116 } 117 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 118 } else { 119 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 120 } 121 if (ret) { 122 hfi1_early_err(&pdev->dev, 123 "Unable to set DMA consistent mask: %d\n", ret); 124 goto bail; 125 } 126 127 pci_set_master(pdev); 128 (void)pci_enable_pcie_error_reporting(pdev); 129 goto done; 130 131 bail: 132 hfi1_pcie_cleanup(pdev); 133 done: 134 return ret; 135 } 136 137 /* 138 * Clean what was done in hfi1_pcie_init() 139 */ 140 void hfi1_pcie_cleanup(struct pci_dev *pdev) 141 { 142 pci_disable_device(pdev); 143 /* 144 * Release regions should be called after the disable. OK to 145 * call if request regions has not been called or failed. 146 */ 147 pci_release_regions(pdev); 148 } 149 150 /* 151 * Do remaining PCIe setup, once dd is allocated, and save away 152 * fields required to re-initialize after a chip reset, or for 153 * various other purposes 154 */ 155 int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev) 156 { 157 unsigned long len; 158 resource_size_t addr; 159 int ret = 0; 160 u32 rcv_array_count; 161 162 addr = pci_resource_start(pdev, 0); 163 len = pci_resource_len(pdev, 0); 164 165 /* 166 * The TXE PIO buffers are at the tail end of the chip space. 167 * Cut them off and map them separately. 168 */ 169 170 /* sanity check vs expectations */ 171 if (len != TXE_PIO_SEND + TXE_PIO_SIZE) { 172 dd_dev_err(dd, "chip PIO range does not match\n"); 173 return -EINVAL; 174 } 175 176 dd->kregbase1 = ioremap_nocache(addr, RCV_ARRAY); 177 if (!dd->kregbase1) { 178 dd_dev_err(dd, "UC mapping of kregbase1 failed\n"); 179 return -ENOMEM; 180 } 181 dd_dev_info(dd, "UC base1: %p for %x\n", dd->kregbase1, RCV_ARRAY); 182 183 /* verify that reads actually work, save revision for reset check */ 184 dd->revision = readq(dd->kregbase1 + CCE_REVISION); 185 if (dd->revision == ~(u64)0) { 186 dd_dev_err(dd, "Cannot read chip CSRs\n"); 187 goto nomem; 188 } 189 190 rcv_array_count = readq(dd->kregbase1 + RCV_ARRAY_CNT); 191 dd_dev_info(dd, "RcvArray count: %u\n", rcv_array_count); 192 dd->base2_start = RCV_ARRAY + rcv_array_count * 8; 193 194 dd->kregbase2 = ioremap_nocache( 195 addr + dd->base2_start, 196 TXE_PIO_SEND - dd->base2_start); 197 if (!dd->kregbase2) { 198 dd_dev_err(dd, "UC mapping of kregbase2 failed\n"); 199 goto nomem; 200 } 201 dd_dev_info(dd, "UC base2: %p for %x\n", dd->kregbase2, 202 TXE_PIO_SEND - dd->base2_start); 203 204 dd->piobase = ioremap_wc(addr + TXE_PIO_SEND, TXE_PIO_SIZE); 205 if (!dd->piobase) { 206 dd_dev_err(dd, "WC mapping of send buffers failed\n"); 207 goto nomem; 208 } 209 dd_dev_info(dd, "WC piobase: %p\n for %x", dd->piobase, TXE_PIO_SIZE); 210 211 dd->physaddr = addr; /* used for io_remap, etc. */ 212 213 /* 214 * Map the chip's RcvArray as write-combining to allow us 215 * to write an entire cacheline worth of entries in one shot. 216 */ 217 dd->rcvarray_wc = ioremap_wc(addr + RCV_ARRAY, 218 rcv_array_count * 8); 219 if (!dd->rcvarray_wc) { 220 dd_dev_err(dd, "WC mapping of receive array failed\n"); 221 goto nomem; 222 } 223 dd_dev_info(dd, "WC RcvArray: %p for %x\n", 224 dd->rcvarray_wc, rcv_array_count * 8); 225 226 dd->flags |= HFI1_PRESENT; /* chip.c CSR routines now work */ 227 return 0; 228 nomem: 229 ret = -ENOMEM; 230 hfi1_pcie_ddcleanup(dd); 231 return ret; 232 } 233 234 /* 235 * Do PCIe cleanup related to dd, after chip-specific cleanup, etc. Just prior 236 * to releasing the dd memory. 237 * Void because all of the core pcie cleanup functions are void. 238 */ 239 void hfi1_pcie_ddcleanup(struct hfi1_devdata *dd) 240 { 241 dd->flags &= ~HFI1_PRESENT; 242 if (dd->kregbase1) 243 iounmap(dd->kregbase1); 244 dd->kregbase1 = NULL; 245 if (dd->kregbase2) 246 iounmap(dd->kregbase2); 247 dd->kregbase2 = NULL; 248 if (dd->rcvarray_wc) 249 iounmap(dd->rcvarray_wc); 250 dd->rcvarray_wc = NULL; 251 if (dd->piobase) 252 iounmap(dd->piobase); 253 dd->piobase = NULL; 254 } 255 256 /* return the PCIe link speed from the given link status */ 257 static u32 extract_speed(u16 linkstat) 258 { 259 u32 speed; 260 261 switch (linkstat & PCI_EXP_LNKSTA_CLS) { 262 default: /* not defined, assume Gen1 */ 263 case PCI_EXP_LNKSTA_CLS_2_5GB: 264 speed = 2500; /* Gen 1, 2.5GHz */ 265 break; 266 case PCI_EXP_LNKSTA_CLS_5_0GB: 267 speed = 5000; /* Gen 2, 5GHz */ 268 break; 269 case PCI_EXP_LNKSTA_CLS_8_0GB: 270 speed = 8000; /* Gen 3, 8GHz */ 271 break; 272 } 273 return speed; 274 } 275 276 /* return the PCIe link speed from the given link status */ 277 static u32 extract_width(u16 linkstat) 278 { 279 return (linkstat & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; 280 } 281 282 /* read the link status and set dd->{lbus_width,lbus_speed,lbus_info} */ 283 static void update_lbus_info(struct hfi1_devdata *dd) 284 { 285 u16 linkstat; 286 int ret; 287 288 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKSTA, &linkstat); 289 if (ret) { 290 dd_dev_err(dd, "Unable to read from PCI config\n"); 291 return; 292 } 293 294 dd->lbus_width = extract_width(linkstat); 295 dd->lbus_speed = extract_speed(linkstat); 296 snprintf(dd->lbus_info, sizeof(dd->lbus_info), 297 "PCIe,%uMHz,x%u", dd->lbus_speed, dd->lbus_width); 298 } 299 300 /* 301 * Read in the current PCIe link width and speed. Find if the link is 302 * Gen3 capable. 303 */ 304 int pcie_speeds(struct hfi1_devdata *dd) 305 { 306 u32 linkcap; 307 struct pci_dev *parent = dd->pcidev->bus->self; 308 int ret; 309 310 if (!pci_is_pcie(dd->pcidev)) { 311 dd_dev_err(dd, "Can't find PCI Express capability!\n"); 312 return -EINVAL; 313 } 314 315 /* find if our max speed is Gen3 and parent supports Gen3 speeds */ 316 dd->link_gen3_capable = 1; 317 318 ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap); 319 if (ret) { 320 dd_dev_err(dd, "Unable to read from PCI config\n"); 321 return ret; 322 } 323 324 if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) { 325 dd_dev_info(dd, 326 "This HFI is not Gen3 capable, max speed 0x%x, need 0x3\n", 327 linkcap & PCI_EXP_LNKCAP_SLS); 328 dd->link_gen3_capable = 0; 329 } 330 331 /* 332 * bus->max_bus_speed is set from the bridge's linkcap Max Link Speed 333 */ 334 if (parent && dd->pcidev->bus->max_bus_speed != PCIE_SPEED_8_0GT) { 335 dd_dev_info(dd, "Parent PCIe bridge does not support Gen3\n"); 336 dd->link_gen3_capable = 0; 337 } 338 339 /* obtain the link width and current speed */ 340 update_lbus_info(dd); 341 342 dd_dev_info(dd, "%s\n", dd->lbus_info); 343 344 return 0; 345 } 346 347 /* 348 * Returns: 349 * - actual number of interrupts allocated or 350 * - error 351 */ 352 int request_msix(struct hfi1_devdata *dd, u32 msireq) 353 { 354 int nvec; 355 356 nvec = pci_alloc_irq_vectors(dd->pcidev, msireq, msireq, PCI_IRQ_MSIX); 357 if (nvec < 0) { 358 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", nvec); 359 return nvec; 360 } 361 362 tune_pcie_caps(dd); 363 364 return nvec; 365 } 366 367 /* restore command and BARs after a reset has wiped them out */ 368 int restore_pci_variables(struct hfi1_devdata *dd) 369 { 370 int ret = 0; 371 372 ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command); 373 if (ret) 374 goto error; 375 376 ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0, 377 dd->pcibar0); 378 if (ret) 379 goto error; 380 381 ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1, 382 dd->pcibar1); 383 if (ret) 384 goto error; 385 386 ret = pci_write_config_dword(dd->pcidev, PCI_ROM_ADDRESS, dd->pci_rom); 387 if (ret) 388 goto error; 389 390 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL, 391 dd->pcie_devctl); 392 if (ret) 393 goto error; 394 395 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL, 396 dd->pcie_lnkctl); 397 if (ret) 398 goto error; 399 400 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL2, 401 dd->pcie_devctl2); 402 if (ret) 403 goto error; 404 405 ret = pci_write_config_dword(dd->pcidev, PCI_CFG_MSIX0, dd->pci_msix0); 406 if (ret) 407 goto error; 408 409 if (pci_find_ext_capability(dd->pcidev, PCI_EXT_CAP_ID_TPH)) { 410 ret = pci_write_config_dword(dd->pcidev, PCIE_CFG_TPH2, 411 dd->pci_tph2); 412 if (ret) 413 goto error; 414 } 415 return 0; 416 417 error: 418 dd_dev_err(dd, "Unable to write to PCI config\n"); 419 return ret; 420 } 421 422 /* Save BARs and command to rewrite after device reset */ 423 int save_pci_variables(struct hfi1_devdata *dd) 424 { 425 int ret = 0; 426 427 ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0, 428 &dd->pcibar0); 429 if (ret) 430 goto error; 431 432 ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1, 433 &dd->pcibar1); 434 if (ret) 435 goto error; 436 437 ret = pci_read_config_dword(dd->pcidev, PCI_ROM_ADDRESS, &dd->pci_rom); 438 if (ret) 439 goto error; 440 441 ret = pci_read_config_word(dd->pcidev, PCI_COMMAND, &dd->pci_command); 442 if (ret) 443 goto error; 444 445 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL, 446 &dd->pcie_devctl); 447 if (ret) 448 goto error; 449 450 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL, 451 &dd->pcie_lnkctl); 452 if (ret) 453 goto error; 454 455 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL2, 456 &dd->pcie_devctl2); 457 if (ret) 458 goto error; 459 460 ret = pci_read_config_dword(dd->pcidev, PCI_CFG_MSIX0, &dd->pci_msix0); 461 if (ret) 462 goto error; 463 464 if (pci_find_ext_capability(dd->pcidev, PCI_EXT_CAP_ID_TPH)) { 465 ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_TPH2, 466 &dd->pci_tph2); 467 if (ret) 468 goto error; 469 } 470 return 0; 471 472 error: 473 dd_dev_err(dd, "Unable to read from PCI config\n"); 474 return ret; 475 } 476 477 /* 478 * BIOS may not set PCIe bus-utilization parameters for best performance. 479 * Check and optionally adjust them to maximize our throughput. 480 */ 481 static int hfi1_pcie_caps; 482 module_param_named(pcie_caps, hfi1_pcie_caps, int, S_IRUGO); 483 MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)"); 484 485 uint aspm_mode = ASPM_MODE_DISABLED; 486 module_param_named(aspm, aspm_mode, uint, S_IRUGO); 487 MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic"); 488 489 static void tune_pcie_caps(struct hfi1_devdata *dd) 490 { 491 struct pci_dev *parent; 492 u16 rc_mpss, rc_mps, ep_mpss, ep_mps; 493 u16 rc_mrrs, ep_mrrs, max_mrrs, ectl; 494 int ret; 495 496 /* 497 * Turn on extended tags in DevCtl in case the BIOS has turned it off 498 * to improve WFR SDMA bandwidth 499 */ 500 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL, &ectl); 501 if ((!ret) && !(ectl & PCI_EXP_DEVCTL_EXT_TAG)) { 502 dd_dev_info(dd, "Enabling PCIe extended tags\n"); 503 ectl |= PCI_EXP_DEVCTL_EXT_TAG; 504 ret = pcie_capability_write_word(dd->pcidev, 505 PCI_EXP_DEVCTL, ectl); 506 if (ret) 507 dd_dev_info(dd, "Unable to write to PCI config\n"); 508 } 509 /* Find out supported and configured values for parent (root) */ 510 parent = dd->pcidev->bus->self; 511 /* 512 * The driver cannot perform the tuning if it does not have 513 * access to the upstream component. 514 */ 515 if (!parent) { 516 dd_dev_info(dd, "Parent not found\n"); 517 return; 518 } 519 if (!pci_is_root_bus(parent->bus)) { 520 dd_dev_info(dd, "Parent not root\n"); 521 return; 522 } 523 if (!pci_is_pcie(parent)) { 524 dd_dev_info(dd, "Parent is not PCI Express capable\n"); 525 return; 526 } 527 if (!pci_is_pcie(dd->pcidev)) { 528 dd_dev_info(dd, "PCI device is not PCI Express capable\n"); 529 return; 530 } 531 rc_mpss = parent->pcie_mpss; 532 rc_mps = ffs(pcie_get_mps(parent)) - 8; 533 /* Find out supported and configured values for endpoint (us) */ 534 ep_mpss = dd->pcidev->pcie_mpss; 535 ep_mps = ffs(pcie_get_mps(dd->pcidev)) - 8; 536 537 /* Find max payload supported by root, endpoint */ 538 if (rc_mpss > ep_mpss) 539 rc_mpss = ep_mpss; 540 541 /* If Supported greater than limit in module param, limit it */ 542 if (rc_mpss > (hfi1_pcie_caps & 7)) 543 rc_mpss = hfi1_pcie_caps & 7; 544 /* If less than (allowed, supported), bump root payload */ 545 if (rc_mpss > rc_mps) { 546 rc_mps = rc_mpss; 547 pcie_set_mps(parent, 128 << rc_mps); 548 } 549 /* If less than (allowed, supported), bump endpoint payload */ 550 if (rc_mpss > ep_mps) { 551 ep_mps = rc_mpss; 552 pcie_set_mps(dd->pcidev, 128 << ep_mps); 553 } 554 555 /* 556 * Now the Read Request size. 557 * No field for max supported, but PCIe spec limits it to 4096, 558 * which is code '5' (log2(4096) - 7) 559 */ 560 max_mrrs = 5; 561 if (max_mrrs > ((hfi1_pcie_caps >> 4) & 7)) 562 max_mrrs = (hfi1_pcie_caps >> 4) & 7; 563 564 max_mrrs = 128 << max_mrrs; 565 rc_mrrs = pcie_get_readrq(parent); 566 ep_mrrs = pcie_get_readrq(dd->pcidev); 567 568 if (max_mrrs > rc_mrrs) { 569 rc_mrrs = max_mrrs; 570 pcie_set_readrq(parent, rc_mrrs); 571 } 572 if (max_mrrs > ep_mrrs) { 573 ep_mrrs = max_mrrs; 574 pcie_set_readrq(dd->pcidev, ep_mrrs); 575 } 576 } 577 578 /* End of PCIe capability tuning */ 579 580 /* 581 * From here through hfi1_pci_err_handler definition is invoked via 582 * PCI error infrastructure, registered via pci 583 */ 584 static pci_ers_result_t 585 pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 586 { 587 struct hfi1_devdata *dd = pci_get_drvdata(pdev); 588 pci_ers_result_t ret = PCI_ERS_RESULT_RECOVERED; 589 590 switch (state) { 591 case pci_channel_io_normal: 592 dd_dev_info(dd, "State Normal, ignoring\n"); 593 break; 594 595 case pci_channel_io_frozen: 596 dd_dev_info(dd, "State Frozen, requesting reset\n"); 597 pci_disable_device(pdev); 598 ret = PCI_ERS_RESULT_NEED_RESET; 599 break; 600 601 case pci_channel_io_perm_failure: 602 if (dd) { 603 dd_dev_info(dd, "State Permanent Failure, disabling\n"); 604 /* no more register accesses! */ 605 dd->flags &= ~HFI1_PRESENT; 606 hfi1_disable_after_error(dd); 607 } 608 /* else early, or other problem */ 609 ret = PCI_ERS_RESULT_DISCONNECT; 610 break; 611 612 default: /* shouldn't happen */ 613 dd_dev_info(dd, "HFI1 PCI errors detected (state %d)\n", 614 state); 615 break; 616 } 617 return ret; 618 } 619 620 static pci_ers_result_t 621 pci_mmio_enabled(struct pci_dev *pdev) 622 { 623 u64 words = 0U; 624 struct hfi1_devdata *dd = pci_get_drvdata(pdev); 625 pci_ers_result_t ret = PCI_ERS_RESULT_RECOVERED; 626 627 if (dd && dd->pport) { 628 words = read_port_cntr(dd->pport, C_RX_WORDS, CNTR_INVALID_VL); 629 if (words == ~0ULL) 630 ret = PCI_ERS_RESULT_NEED_RESET; 631 dd_dev_info(dd, 632 "HFI1 mmio_enabled function called, read wordscntr %llx, returning %d\n", 633 words, ret); 634 } 635 return ret; 636 } 637 638 static pci_ers_result_t 639 pci_slot_reset(struct pci_dev *pdev) 640 { 641 struct hfi1_devdata *dd = pci_get_drvdata(pdev); 642 643 dd_dev_info(dd, "HFI1 slot_reset function called, ignored\n"); 644 return PCI_ERS_RESULT_CAN_RECOVER; 645 } 646 647 static void 648 pci_resume(struct pci_dev *pdev) 649 { 650 struct hfi1_devdata *dd = pci_get_drvdata(pdev); 651 652 dd_dev_info(dd, "HFI1 resume function called\n"); 653 pci_cleanup_aer_uncorrect_error_status(pdev); 654 /* 655 * Running jobs will fail, since it's asynchronous 656 * unlike sysfs-requested reset. Better than 657 * doing nothing. 658 */ 659 hfi1_init(dd, 1); /* same as re-init after reset */ 660 } 661 662 const struct pci_error_handlers hfi1_pci_err_handler = { 663 .error_detected = pci_error_detected, 664 .mmio_enabled = pci_mmio_enabled, 665 .slot_reset = pci_slot_reset, 666 .resume = pci_resume, 667 }; 668 669 /*============================================================================*/ 670 /* PCIe Gen3 support */ 671 672 /* 673 * This code is separated out because it is expected to be removed in the 674 * final shipping product. If not, then it will be revisited and items 675 * will be moved to more standard locations. 676 */ 677 678 /* ASIC_PCI_SD_HOST_STATUS.FW_DNLD_STS field values */ 679 #define DL_STATUS_HFI0 0x1 /* hfi0 firmware download complete */ 680 #define DL_STATUS_HFI1 0x2 /* hfi1 firmware download complete */ 681 #define DL_STATUS_BOTH 0x3 /* hfi0 and hfi1 firmware download complete */ 682 683 /* ASIC_PCI_SD_HOST_STATUS.FW_DNLD_ERR field values */ 684 #define DL_ERR_NONE 0x0 /* no error */ 685 #define DL_ERR_SWAP_PARITY 0x1 /* parity error in SerDes interrupt */ 686 /* or response data */ 687 #define DL_ERR_DISABLED 0x2 /* hfi disabled */ 688 #define DL_ERR_SECURITY 0x3 /* security check failed */ 689 #define DL_ERR_SBUS 0x4 /* SBus status error */ 690 #define DL_ERR_XFR_PARITY 0x5 /* parity error during ROM transfer*/ 691 692 /* gasket block secondary bus reset delay */ 693 #define SBR_DELAY_US 200000 /* 200ms */ 694 695 static uint pcie_target = 3; 696 module_param(pcie_target, uint, S_IRUGO); 697 MODULE_PARM_DESC(pcie_target, "PCIe target speed (0 skip, 1-3 Gen1-3)"); 698 699 static uint pcie_force; 700 module_param(pcie_force, uint, S_IRUGO); 701 MODULE_PARM_DESC(pcie_force, "Force driver to do a PCIe firmware download even if already at target speed"); 702 703 static uint pcie_retry = 5; 704 module_param(pcie_retry, uint, S_IRUGO); 705 MODULE_PARM_DESC(pcie_retry, "Driver will try this many times to reach requested speed"); 706 707 #define UNSET_PSET 255 708 #define DEFAULT_DISCRETE_PSET 2 /* discrete HFI */ 709 #define DEFAULT_MCP_PSET 6 /* MCP HFI */ 710 static uint pcie_pset = UNSET_PSET; 711 module_param(pcie_pset, uint, S_IRUGO); 712 MODULE_PARM_DESC(pcie_pset, "PCIe Eq Pset value to use, range is 0-10"); 713 714 static uint pcie_ctle = 3; /* discrete on, integrated on */ 715 module_param(pcie_ctle, uint, S_IRUGO); 716 MODULE_PARM_DESC(pcie_ctle, "PCIe static CTLE mode, bit 0 - discrete on/off, bit 1 - integrated on/off"); 717 718 /* equalization columns */ 719 #define PREC 0 720 #define ATTN 1 721 #define POST 2 722 723 /* discrete silicon preliminary equalization values */ 724 static const u8 discrete_preliminary_eq[11][3] = { 725 /* prec attn post */ 726 { 0x00, 0x00, 0x12 }, /* p0 */ 727 { 0x00, 0x00, 0x0c }, /* p1 */ 728 { 0x00, 0x00, 0x0f }, /* p2 */ 729 { 0x00, 0x00, 0x09 }, /* p3 */ 730 { 0x00, 0x00, 0x00 }, /* p4 */ 731 { 0x06, 0x00, 0x00 }, /* p5 */ 732 { 0x09, 0x00, 0x00 }, /* p6 */ 733 { 0x06, 0x00, 0x0f }, /* p7 */ 734 { 0x09, 0x00, 0x09 }, /* p8 */ 735 { 0x0c, 0x00, 0x00 }, /* p9 */ 736 { 0x00, 0x00, 0x18 }, /* p10 */ 737 }; 738 739 /* integrated silicon preliminary equalization values */ 740 static const u8 integrated_preliminary_eq[11][3] = { 741 /* prec attn post */ 742 { 0x00, 0x1e, 0x07 }, /* p0 */ 743 { 0x00, 0x1e, 0x05 }, /* p1 */ 744 { 0x00, 0x1e, 0x06 }, /* p2 */ 745 { 0x00, 0x1e, 0x04 }, /* p3 */ 746 { 0x00, 0x1e, 0x00 }, /* p4 */ 747 { 0x03, 0x1e, 0x00 }, /* p5 */ 748 { 0x04, 0x1e, 0x00 }, /* p6 */ 749 { 0x03, 0x1e, 0x06 }, /* p7 */ 750 { 0x03, 0x1e, 0x04 }, /* p8 */ 751 { 0x05, 0x1e, 0x00 }, /* p9 */ 752 { 0x00, 0x1e, 0x0a }, /* p10 */ 753 }; 754 755 static const u8 discrete_ctle_tunings[11][4] = { 756 /* DC LF HF BW */ 757 { 0x48, 0x0b, 0x04, 0x04 }, /* p0 */ 758 { 0x60, 0x05, 0x0f, 0x0a }, /* p1 */ 759 { 0x50, 0x09, 0x06, 0x06 }, /* p2 */ 760 { 0x68, 0x05, 0x0f, 0x0a }, /* p3 */ 761 { 0x80, 0x05, 0x0f, 0x0a }, /* p4 */ 762 { 0x70, 0x05, 0x0f, 0x0a }, /* p5 */ 763 { 0x68, 0x05, 0x0f, 0x0a }, /* p6 */ 764 { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */ 765 { 0x48, 0x09, 0x06, 0x06 }, /* p8 */ 766 { 0x60, 0x05, 0x0f, 0x0a }, /* p9 */ 767 { 0x38, 0x0f, 0x00, 0x00 }, /* p10 */ 768 }; 769 770 static const u8 integrated_ctle_tunings[11][4] = { 771 /* DC LF HF BW */ 772 { 0x38, 0x0f, 0x00, 0x00 }, /* p0 */ 773 { 0x38, 0x0f, 0x00, 0x00 }, /* p1 */ 774 { 0x38, 0x0f, 0x00, 0x00 }, /* p2 */ 775 { 0x38, 0x0f, 0x00, 0x00 }, /* p3 */ 776 { 0x58, 0x0a, 0x05, 0x05 }, /* p4 */ 777 { 0x48, 0x0a, 0x05, 0x05 }, /* p5 */ 778 { 0x40, 0x0a, 0x05, 0x05 }, /* p6 */ 779 { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */ 780 { 0x38, 0x0f, 0x00, 0x00 }, /* p8 */ 781 { 0x38, 0x09, 0x06, 0x06 }, /* p9 */ 782 { 0x38, 0x0e, 0x01, 0x01 }, /* p10 */ 783 }; 784 785 /* helper to format the value to write to hardware */ 786 #define eq_value(pre, curr, post) \ 787 ((((u32)(pre)) << \ 788 PCIE_CFG_REG_PL102_GEN3_EQ_PRE_CURSOR_PSET_SHIFT) \ 789 | (((u32)(curr)) << PCIE_CFG_REG_PL102_GEN3_EQ_CURSOR_PSET_SHIFT) \ 790 | (((u32)(post)) << \ 791 PCIE_CFG_REG_PL102_GEN3_EQ_POST_CURSOR_PSET_SHIFT)) 792 793 /* 794 * Load the given EQ preset table into the PCIe hardware. 795 */ 796 static int load_eq_table(struct hfi1_devdata *dd, const u8 eq[11][3], u8 fs, 797 u8 div) 798 { 799 struct pci_dev *pdev = dd->pcidev; 800 u32 hit_error = 0; 801 u32 violation; 802 u32 i; 803 u8 c_minus1, c0, c_plus1; 804 int ret; 805 806 for (i = 0; i < 11; i++) { 807 /* set index */ 808 pci_write_config_dword(pdev, PCIE_CFG_REG_PL103, i); 809 /* write the value */ 810 c_minus1 = eq[i][PREC] / div; 811 c0 = fs - (eq[i][PREC] / div) - (eq[i][POST] / div); 812 c_plus1 = eq[i][POST] / div; 813 pci_write_config_dword(pdev, PCIE_CFG_REG_PL102, 814 eq_value(c_minus1, c0, c_plus1)); 815 /* check if these coefficients violate EQ rules */ 816 ret = pci_read_config_dword(dd->pcidev, 817 PCIE_CFG_REG_PL105, &violation); 818 if (ret) { 819 dd_dev_err(dd, "Unable to read from PCI config\n"); 820 hit_error = 1; 821 break; 822 } 823 824 if (violation 825 & PCIE_CFG_REG_PL105_GEN3_EQ_VIOLATE_COEF_RULES_SMASK){ 826 if (hit_error == 0) { 827 dd_dev_err(dd, 828 "Gen3 EQ Table Coefficient rule violations\n"); 829 dd_dev_err(dd, " prec attn post\n"); 830 } 831 dd_dev_err(dd, " p%02d: %02x %02x %02x\n", 832 i, (u32)eq[i][0], (u32)eq[i][1], 833 (u32)eq[i][2]); 834 dd_dev_err(dd, " %02x %02x %02x\n", 835 (u32)c_minus1, (u32)c0, (u32)c_plus1); 836 hit_error = 1; 837 } 838 } 839 if (hit_error) 840 return -EINVAL; 841 return 0; 842 } 843 844 /* 845 * Steps to be done after the PCIe firmware is downloaded and 846 * before the SBR for the Pcie Gen3. 847 * The SBus resource is already being held. 848 */ 849 static void pcie_post_steps(struct hfi1_devdata *dd) 850 { 851 int i; 852 853 set_sbus_fast_mode(dd); 854 /* 855 * Write to the PCIe PCSes to set the G3_LOCKED_NEXT bits to 1. 856 * This avoids a spurious framing error that can otherwise be 857 * generated by the MAC layer. 858 * 859 * Use individual addresses since no broadcast is set up. 860 */ 861 for (i = 0; i < NUM_PCIE_SERDES; i++) { 862 sbus_request(dd, pcie_pcs_addrs[dd->hfi1_id][i], 863 0x03, WRITE_SBUS_RECEIVER, 0x00022132); 864 } 865 866 clear_sbus_fast_mode(dd); 867 } 868 869 /* 870 * Trigger a secondary bus reset (SBR) on ourselves using our parent. 871 * 872 * Based on pci_parent_bus_reset() which is not exported by the 873 * kernel core. 874 */ 875 static int trigger_sbr(struct hfi1_devdata *dd) 876 { 877 struct pci_dev *dev = dd->pcidev; 878 struct pci_dev *pdev; 879 880 /* need a parent */ 881 if (!dev->bus->self) { 882 dd_dev_err(dd, "%s: no parent device\n", __func__); 883 return -ENOTTY; 884 } 885 886 /* should not be anyone else on the bus */ 887 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 888 if (pdev != dev) { 889 dd_dev_err(dd, 890 "%s: another device is on the same bus\n", 891 __func__); 892 return -ENOTTY; 893 } 894 895 /* 896 * A secondary bus reset (SBR) issues a hot reset to our device. 897 * The following routine does a 1s wait after the reset is dropped 898 * per PCI Trhfa (recovery time). PCIe 3.0 section 6.6.1 - 899 * Conventional Reset, paragraph 3, line 35 also says that a 1s 900 * delay after a reset is required. Per spec requirements, 901 * the link is either working or not after that point. 902 */ 903 return pci_reset_bus(dev); 904 } 905 906 /* 907 * Write the given gasket interrupt register. 908 */ 909 static void write_gasket_interrupt(struct hfi1_devdata *dd, int index, 910 u16 code, u16 data) 911 { 912 write_csr(dd, ASIC_PCIE_SD_INTRPT_LIST + (index * 8), 913 (((u64)code << ASIC_PCIE_SD_INTRPT_LIST_INTRPT_CODE_SHIFT) | 914 ((u64)data << ASIC_PCIE_SD_INTRPT_LIST_INTRPT_DATA_SHIFT))); 915 } 916 917 /* 918 * Tell the gasket logic how to react to the reset. 919 */ 920 static void arm_gasket_logic(struct hfi1_devdata *dd) 921 { 922 u64 reg; 923 924 reg = (((u64)1 << dd->hfi1_id) << 925 ASIC_PCIE_SD_HOST_CMD_INTRPT_CMD_SHIFT) | 926 ((u64)pcie_serdes_broadcast[dd->hfi1_id] << 927 ASIC_PCIE_SD_HOST_CMD_SBUS_RCVR_ADDR_SHIFT | 928 ASIC_PCIE_SD_HOST_CMD_SBR_MODE_SMASK | 929 ((u64)SBR_DELAY_US & ASIC_PCIE_SD_HOST_CMD_TIMER_MASK) << 930 ASIC_PCIE_SD_HOST_CMD_TIMER_SHIFT); 931 write_csr(dd, ASIC_PCIE_SD_HOST_CMD, reg); 932 /* read back to push the write */ 933 read_csr(dd, ASIC_PCIE_SD_HOST_CMD); 934 } 935 936 /* 937 * CCE_PCIE_CTRL long name helpers 938 * We redefine these shorter macros to use in the code while leaving 939 * chip_registers.h to be autogenerated from the hardware spec. 940 */ 941 #define LANE_BUNDLE_MASK CCE_PCIE_CTRL_PCIE_LANE_BUNDLE_MASK 942 #define LANE_BUNDLE_SHIFT CCE_PCIE_CTRL_PCIE_LANE_BUNDLE_SHIFT 943 #define LANE_DELAY_MASK CCE_PCIE_CTRL_PCIE_LANE_DELAY_MASK 944 #define LANE_DELAY_SHIFT CCE_PCIE_CTRL_PCIE_LANE_DELAY_SHIFT 945 #define MARGIN_OVERWRITE_ENABLE_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_OVERWRITE_ENABLE_SHIFT 946 #define MARGIN_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_SHIFT 947 #define MARGIN_G1_G2_OVERWRITE_MASK CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_OVERWRITE_ENABLE_MASK 948 #define MARGIN_G1_G2_OVERWRITE_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_OVERWRITE_ENABLE_SHIFT 949 #define MARGIN_GEN1_GEN2_MASK CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_MASK 950 #define MARGIN_GEN1_GEN2_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_SHIFT 951 952 /* 953 * Write xmt_margin for full-swing (WFR-B) or half-swing (WFR-C). 954 */ 955 static void write_xmt_margin(struct hfi1_devdata *dd, const char *fname) 956 { 957 u64 pcie_ctrl; 958 u64 xmt_margin; 959 u64 xmt_margin_oe; 960 u64 lane_delay; 961 u64 lane_bundle; 962 963 pcie_ctrl = read_csr(dd, CCE_PCIE_CTRL); 964 965 /* 966 * For Discrete, use full-swing. 967 * - PCIe TX defaults to full-swing. 968 * Leave this register as default. 969 * For Integrated, use half-swing 970 * - Copy xmt_margin and xmt_margin_oe 971 * from Gen1/Gen2 to Gen3. 972 */ 973 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL1) { /* integrated */ 974 /* extract initial fields */ 975 xmt_margin = (pcie_ctrl >> MARGIN_GEN1_GEN2_SHIFT) 976 & MARGIN_GEN1_GEN2_MASK; 977 xmt_margin_oe = (pcie_ctrl >> MARGIN_G1_G2_OVERWRITE_SHIFT) 978 & MARGIN_G1_G2_OVERWRITE_MASK; 979 lane_delay = (pcie_ctrl >> LANE_DELAY_SHIFT) & LANE_DELAY_MASK; 980 lane_bundle = (pcie_ctrl >> LANE_BUNDLE_SHIFT) 981 & LANE_BUNDLE_MASK; 982 983 /* 984 * For A0, EFUSE values are not set. Override with the 985 * correct values. 986 */ 987 if (is_ax(dd)) { 988 /* 989 * xmt_margin and OverwiteEnabel should be the 990 * same for Gen1/Gen2 and Gen3 991 */ 992 xmt_margin = 0x5; 993 xmt_margin_oe = 0x1; 994 lane_delay = 0xF; /* Delay 240ns. */ 995 lane_bundle = 0x0; /* Set to 1 lane. */ 996 } 997 998 /* overwrite existing values */ 999 pcie_ctrl = (xmt_margin << MARGIN_GEN1_GEN2_SHIFT) 1000 | (xmt_margin_oe << MARGIN_G1_G2_OVERWRITE_SHIFT) 1001 | (xmt_margin << MARGIN_SHIFT) 1002 | (xmt_margin_oe << MARGIN_OVERWRITE_ENABLE_SHIFT) 1003 | (lane_delay << LANE_DELAY_SHIFT) 1004 | (lane_bundle << LANE_BUNDLE_SHIFT); 1005 1006 write_csr(dd, CCE_PCIE_CTRL, pcie_ctrl); 1007 } 1008 1009 dd_dev_dbg(dd, "%s: program XMT margin, CcePcieCtrl 0x%llx\n", 1010 fname, pcie_ctrl); 1011 } 1012 1013 /* 1014 * Do all the steps needed to transition the PCIe link to Gen3 speed. 1015 */ 1016 int do_pcie_gen3_transition(struct hfi1_devdata *dd) 1017 { 1018 struct pci_dev *parent = dd->pcidev->bus->self; 1019 u64 fw_ctrl; 1020 u64 reg, therm; 1021 u32 reg32, fs, lf; 1022 u32 status, err; 1023 int ret; 1024 int do_retry, retry_count = 0; 1025 int intnum = 0; 1026 uint default_pset; 1027 uint pset = pcie_pset; 1028 u16 target_vector, target_speed; 1029 u16 lnkctl2, vendor; 1030 u8 div; 1031 const u8 (*eq)[3]; 1032 const u8 (*ctle_tunings)[4]; 1033 uint static_ctle_mode; 1034 int return_error = 0; 1035 1036 /* PCIe Gen3 is for the ASIC only */ 1037 if (dd->icode != ICODE_RTL_SILICON) 1038 return 0; 1039 1040 if (pcie_target == 1) { /* target Gen1 */ 1041 target_vector = PCI_EXP_LNKCTL2_TLS_2_5GT; 1042 target_speed = 2500; 1043 } else if (pcie_target == 2) { /* target Gen2 */ 1044 target_vector = PCI_EXP_LNKCTL2_TLS_5_0GT; 1045 target_speed = 5000; 1046 } else if (pcie_target == 3) { /* target Gen3 */ 1047 target_vector = PCI_EXP_LNKCTL2_TLS_8_0GT; 1048 target_speed = 8000; 1049 } else { 1050 /* off or invalid target - skip */ 1051 dd_dev_info(dd, "%s: Skipping PCIe transition\n", __func__); 1052 return 0; 1053 } 1054 1055 /* if already at target speed, done (unless forced) */ 1056 if (dd->lbus_speed == target_speed) { 1057 dd_dev_info(dd, "%s: PCIe already at gen%d, %s\n", __func__, 1058 pcie_target, 1059 pcie_force ? "re-doing anyway" : "skipping"); 1060 if (!pcie_force) 1061 return 0; 1062 } 1063 1064 /* 1065 * The driver cannot do the transition if it has no access to the 1066 * upstream component 1067 */ 1068 if (!parent) { 1069 dd_dev_info(dd, "%s: No upstream, Can't do gen3 transition\n", 1070 __func__); 1071 return 0; 1072 } 1073 1074 /* 1075 * Do the Gen3 transition. Steps are those of the PCIe Gen3 1076 * recipe. 1077 */ 1078 1079 /* step 1: pcie link working in gen1/gen2 */ 1080 1081 /* step 2: if either side is not capable of Gen3, done */ 1082 if (pcie_target == 3 && !dd->link_gen3_capable) { 1083 dd_dev_err(dd, "The PCIe link is not Gen3 capable\n"); 1084 ret = -ENOSYS; 1085 goto done_no_mutex; 1086 } 1087 1088 /* hold the SBus resource across the firmware download and SBR */ 1089 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT); 1090 if (ret) { 1091 dd_dev_err(dd, "%s: unable to acquire SBus resource\n", 1092 __func__); 1093 return ret; 1094 } 1095 1096 /* make sure thermal polling is not causing interrupts */ 1097 therm = read_csr(dd, ASIC_CFG_THERM_POLL_EN); 1098 if (therm) { 1099 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0); 1100 msleep(100); 1101 dd_dev_info(dd, "%s: Disabled therm polling\n", 1102 __func__); 1103 } 1104 1105 retry: 1106 /* the SBus download will reset the spico for thermal */ 1107 1108 /* step 3: download SBus Master firmware */ 1109 /* step 4: download PCIe Gen3 SerDes firmware */ 1110 dd_dev_info(dd, "%s: downloading firmware\n", __func__); 1111 ret = load_pcie_firmware(dd); 1112 if (ret) { 1113 /* do not proceed if the firmware cannot be downloaded */ 1114 return_error = 1; 1115 goto done; 1116 } 1117 1118 /* step 5: set up device parameter settings */ 1119 dd_dev_info(dd, "%s: setting PCIe registers\n", __func__); 1120 1121 /* 1122 * PcieCfgSpcie1 - Link Control 3 1123 * Leave at reset value. No need to set PerfEq - link equalization 1124 * will be performed automatically after the SBR when the target 1125 * speed is 8GT/s. 1126 */ 1127 1128 /* clear all 16 per-lane error bits (PCIe: Lane Error Status) */ 1129 pci_write_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, 0xffff); 1130 1131 /* step 5a: Set Synopsys Port Logic registers */ 1132 1133 /* 1134 * PcieCfgRegPl2 - Port Force Link 1135 * 1136 * Set the low power field to 0x10 to avoid unnecessary power 1137 * management messages. All other fields are zero. 1138 */ 1139 reg32 = 0x10ul << PCIE_CFG_REG_PL2_LOW_PWR_ENT_CNT_SHIFT; 1140 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL2, reg32); 1141 1142 /* 1143 * PcieCfgRegPl100 - Gen3 Control 1144 * 1145 * turn off PcieCfgRegPl100.Gen3ZRxDcNonCompl 1146 * turn on PcieCfgRegPl100.EqEieosCnt 1147 * Everything else zero. 1148 */ 1149 reg32 = PCIE_CFG_REG_PL100_EQ_EIEOS_CNT_SMASK; 1150 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL100, reg32); 1151 1152 /* 1153 * PcieCfgRegPl101 - Gen3 EQ FS and LF 1154 * PcieCfgRegPl102 - Gen3 EQ Presets to Coefficients Mapping 1155 * PcieCfgRegPl103 - Gen3 EQ Preset Index 1156 * PcieCfgRegPl105 - Gen3 EQ Status 1157 * 1158 * Give initial EQ settings. 1159 */ 1160 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL0) { /* discrete */ 1161 /* 1000mV, FS=24, LF = 8 */ 1162 fs = 24; 1163 lf = 8; 1164 div = 3; 1165 eq = discrete_preliminary_eq; 1166 default_pset = DEFAULT_DISCRETE_PSET; 1167 ctle_tunings = discrete_ctle_tunings; 1168 /* bit 0 - discrete on/off */ 1169 static_ctle_mode = pcie_ctle & 0x1; 1170 } else { 1171 /* 400mV, FS=29, LF = 9 */ 1172 fs = 29; 1173 lf = 9; 1174 div = 1; 1175 eq = integrated_preliminary_eq; 1176 default_pset = DEFAULT_MCP_PSET; 1177 ctle_tunings = integrated_ctle_tunings; 1178 /* bit 1 - integrated on/off */ 1179 static_ctle_mode = (pcie_ctle >> 1) & 0x1; 1180 } 1181 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101, 1182 (fs << 1183 PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_FS_SHIFT) | 1184 (lf << 1185 PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_LF_SHIFT)); 1186 ret = load_eq_table(dd, eq, fs, div); 1187 if (ret) 1188 goto done; 1189 1190 /* 1191 * PcieCfgRegPl106 - Gen3 EQ Control 1192 * 1193 * Set Gen3EqPsetReqVec, leave other fields 0. 1194 */ 1195 if (pset == UNSET_PSET) 1196 pset = default_pset; 1197 if (pset > 10) { /* valid range is 0-10, inclusive */ 1198 dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n", 1199 __func__, pset, default_pset); 1200 pset = default_pset; 1201 } 1202 dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset); 1203 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106, 1204 ((1 << pset) << 1205 PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) | 1206 PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK | 1207 PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK); 1208 1209 /* 1210 * step 5b: Do post firmware download steps via SBus 1211 */ 1212 dd_dev_info(dd, "%s: doing pcie post steps\n", __func__); 1213 pcie_post_steps(dd); 1214 1215 /* 1216 * step 5c: Program gasket interrupts 1217 */ 1218 /* set the Rx Bit Rate to REFCLK ratio */ 1219 write_gasket_interrupt(dd, intnum++, 0x0006, 0x0050); 1220 /* disable pCal for PCIe Gen3 RX equalization */ 1221 /* select adaptive or static CTLE */ 1222 write_gasket_interrupt(dd, intnum++, 0x0026, 1223 0x5b01 | (static_ctle_mode << 3)); 1224 /* 1225 * Enable iCal for PCIe Gen3 RX equalization, and set which 1226 * evaluation of RX_EQ_EVAL will launch the iCal procedure. 1227 */ 1228 write_gasket_interrupt(dd, intnum++, 0x0026, 0x5202); 1229 1230 if (static_ctle_mode) { 1231 /* apply static CTLE tunings */ 1232 u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw; 1233 1234 pcie_dc = ctle_tunings[pset][0]; 1235 pcie_lf = ctle_tunings[pset][1]; 1236 pcie_hf = ctle_tunings[pset][2]; 1237 pcie_bw = ctle_tunings[pset][3]; 1238 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc); 1239 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf); 1240 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf); 1241 write_gasket_interrupt(dd, intnum++, 0x0026, 0x5500 | pcie_bw); 1242 } 1243 1244 /* terminate list */ 1245 write_gasket_interrupt(dd, intnum++, 0x0000, 0x0000); 1246 1247 /* 1248 * step 5d: program XMT margin 1249 */ 1250 write_xmt_margin(dd, __func__); 1251 1252 /* 1253 * step 5e: disable active state power management (ASPM). It 1254 * will be enabled if required later 1255 */ 1256 dd_dev_info(dd, "%s: clearing ASPM\n", __func__); 1257 aspm_hw_disable_l1(dd); 1258 1259 /* 1260 * step 5f: clear DirectSpeedChange 1261 * PcieCfgRegPl67.DirectSpeedChange must be zero to prevent the 1262 * change in the speed target from starting before we are ready. 1263 * This field defaults to 0 and we are not changing it, so nothing 1264 * needs to be done. 1265 */ 1266 1267 /* step 5g: Set target link speed */ 1268 /* 1269 * Set target link speed to be target on both device and parent. 1270 * On setting the parent: Some system BIOSs "helpfully" set the 1271 * parent target speed to Gen2 to match the ASIC's initial speed. 1272 * We can set the target Gen3 because we have already checked 1273 * that it is Gen3 capable earlier. 1274 */ 1275 dd_dev_info(dd, "%s: setting parent target link speed\n", __func__); 1276 ret = pcie_capability_read_word(parent, PCI_EXP_LNKCTL2, &lnkctl2); 1277 if (ret) { 1278 dd_dev_err(dd, "Unable to read from PCI config\n"); 1279 return_error = 1; 1280 goto done; 1281 } 1282 1283 dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__, 1284 (u32)lnkctl2); 1285 /* only write to parent if target is not as high as ours */ 1286 if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) { 1287 lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; 1288 lnkctl2 |= target_vector; 1289 dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__, 1290 (u32)lnkctl2); 1291 ret = pcie_capability_write_word(parent, 1292 PCI_EXP_LNKCTL2, lnkctl2); 1293 if (ret) { 1294 dd_dev_err(dd, "Unable to write to PCI config\n"); 1295 return_error = 1; 1296 goto done; 1297 } 1298 } else { 1299 dd_dev_info(dd, "%s: ..target speed is OK\n", __func__); 1300 } 1301 1302 dd_dev_info(dd, "%s: setting target link speed\n", __func__); 1303 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2); 1304 if (ret) { 1305 dd_dev_err(dd, "Unable to read from PCI config\n"); 1306 return_error = 1; 1307 goto done; 1308 } 1309 1310 dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__, 1311 (u32)lnkctl2); 1312 lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; 1313 lnkctl2 |= target_vector; 1314 dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__, 1315 (u32)lnkctl2); 1316 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2); 1317 if (ret) { 1318 dd_dev_err(dd, "Unable to write to PCI config\n"); 1319 return_error = 1; 1320 goto done; 1321 } 1322 1323 /* step 5h: arm gasket logic */ 1324 /* hold DC in reset across the SBR */ 1325 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK); 1326 (void)read_csr(dd, CCE_DC_CTRL); /* DC reset hold */ 1327 /* save firmware control across the SBR */ 1328 fw_ctrl = read_csr(dd, MISC_CFG_FW_CTRL); 1329 1330 dd_dev_info(dd, "%s: arming gasket logic\n", __func__); 1331 arm_gasket_logic(dd); 1332 1333 /* 1334 * step 6: quiesce PCIe link 1335 * The chip has already been reset, so there will be no traffic 1336 * from the chip. Linux has no easy way to enforce that it will 1337 * not try to access the device, so we just need to hope it doesn't 1338 * do it while we are doing the reset. 1339 */ 1340 1341 /* 1342 * step 7: initiate the secondary bus reset (SBR) 1343 * step 8: hardware brings the links back up 1344 * step 9: wait for link speed transition to be complete 1345 */ 1346 dd_dev_info(dd, "%s: calling trigger_sbr\n", __func__); 1347 ret = trigger_sbr(dd); 1348 if (ret) 1349 goto done; 1350 1351 /* step 10: decide what to do next */ 1352 1353 /* check if we can read PCI space */ 1354 ret = pci_read_config_word(dd->pcidev, PCI_VENDOR_ID, &vendor); 1355 if (ret) { 1356 dd_dev_info(dd, 1357 "%s: read of VendorID failed after SBR, err %d\n", 1358 __func__, ret); 1359 return_error = 1; 1360 goto done; 1361 } 1362 if (vendor == 0xffff) { 1363 dd_dev_info(dd, "%s: VendorID is all 1s after SBR\n", __func__); 1364 return_error = 1; 1365 ret = -EIO; 1366 goto done; 1367 } 1368 1369 /* restore PCI space registers we know were reset */ 1370 dd_dev_info(dd, "%s: calling restore_pci_variables\n", __func__); 1371 ret = restore_pci_variables(dd); 1372 if (ret) { 1373 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 1374 __func__); 1375 return_error = 1; 1376 goto done; 1377 } 1378 1379 /* restore firmware control */ 1380 write_csr(dd, MISC_CFG_FW_CTRL, fw_ctrl); 1381 1382 /* 1383 * Check the gasket block status. 1384 * 1385 * This is the first CSR read after the SBR. If the read returns 1386 * all 1s (fails), the link did not make it back. 1387 * 1388 * Once we're sure we can read and write, clear the DC reset after 1389 * the SBR. Then check for any per-lane errors. Then look over 1390 * the status. 1391 */ 1392 reg = read_csr(dd, ASIC_PCIE_SD_HOST_STATUS); 1393 dd_dev_info(dd, "%s: gasket block status: 0x%llx\n", __func__, reg); 1394 if (reg == ~0ull) { /* PCIe read failed/timeout */ 1395 dd_dev_err(dd, "SBR failed - unable to read from device\n"); 1396 return_error = 1; 1397 ret = -ENOSYS; 1398 goto done; 1399 } 1400 1401 /* clear the DC reset */ 1402 write_csr(dd, CCE_DC_CTRL, 0); 1403 1404 /* Set the LED off */ 1405 setextled(dd, 0); 1406 1407 /* check for any per-lane errors */ 1408 ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, ®32); 1409 if (ret) { 1410 dd_dev_err(dd, "Unable to read from PCI config\n"); 1411 return_error = 1; 1412 goto done; 1413 } 1414 1415 dd_dev_info(dd, "%s: per-lane errors: 0x%x\n", __func__, reg32); 1416 1417 /* extract status, look for our HFI */ 1418 status = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_SHIFT) 1419 & ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_MASK; 1420 if ((status & (1 << dd->hfi1_id)) == 0) { 1421 dd_dev_err(dd, 1422 "%s: gasket status 0x%x, expecting 0x%x\n", 1423 __func__, status, 1 << dd->hfi1_id); 1424 ret = -EIO; 1425 goto done; 1426 } 1427 1428 /* extract error */ 1429 err = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_SHIFT) 1430 & ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_MASK; 1431 if (err) { 1432 dd_dev_err(dd, "%s: gasket error %d\n", __func__, err); 1433 ret = -EIO; 1434 goto done; 1435 } 1436 1437 /* update our link information cache */ 1438 update_lbus_info(dd); 1439 dd_dev_info(dd, "%s: new speed and width: %s\n", __func__, 1440 dd->lbus_info); 1441 1442 if (dd->lbus_speed != target_speed) { /* not target */ 1443 /* maybe retry */ 1444 do_retry = retry_count < pcie_retry; 1445 dd_dev_err(dd, "PCIe link speed did not switch to Gen%d%s\n", 1446 pcie_target, do_retry ? ", retrying" : ""); 1447 retry_count++; 1448 if (do_retry) { 1449 msleep(100); /* allow time to settle */ 1450 goto retry; 1451 } 1452 ret = -EIO; 1453 } 1454 1455 done: 1456 if (therm) { 1457 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1); 1458 msleep(100); 1459 dd_dev_info(dd, "%s: Re-enable therm polling\n", 1460 __func__); 1461 } 1462 release_chip_resource(dd, CR_SBUS); 1463 done_no_mutex: 1464 /* return no error if it is OK to be at current speed */ 1465 if (ret && !return_error) { 1466 dd_dev_err(dd, "Proceeding at current speed PCIe speed\n"); 1467 ret = 0; 1468 } 1469 1470 dd_dev_info(dd, "%s: done\n", __func__); 1471 return ret; 1472 } 1473