1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright (C) 2007-2015, 2018-2024 Intel Corporation 4 * Copyright (C) 2013-2015 Intel Mobile Communications GmbH 5 * Copyright (C) 2016-2017 Intel Deutschland GmbH 6 */ 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/debugfs.h> 10 #include <linux/sched.h> 11 #include <linux/bitops.h> 12 #include <linux/gfp.h> 13 #include <linux/vmalloc.h> 14 #include <linux/module.h> 15 #include <linux/wait.h> 16 #include <linux/seq_file.h> 17 18 #include "iwl-drv.h" 19 #include "iwl-trans.h" 20 #include "iwl-csr.h" 21 #include "iwl-prph.h" 22 #include "iwl-scd.h" 23 #include "iwl-agn-hw.h" 24 #include "fw/error-dump.h" 25 #include "fw/dbg.h" 26 #include "fw/api/tx.h" 27 #include "fw/acpi.h" 28 #include "fw/api/tx.h" 29 #include "mei/iwl-mei.h" 30 #include "internal.h" 31 #include "iwl-fh.h" 32 #include "pcie/iwl-context-info-v2.h" 33 #include "pcie/utils.h" 34 35 /* extended range in FW SRAM */ 36 #define IWL_FW_MEM_EXTENDED_START 0x40000 37 #define IWL_FW_MEM_EXTENDED_END 0x57FFF 38 39 int iwl_trans_pcie_sw_reset(struct iwl_trans *trans, bool retake_ownership) 40 { 41 /* Reset entire device - do controller reset (results in SHRD_HW_RST) */ 42 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { 43 iwl_set_bit(trans, CSR_GP_CNTRL, 44 CSR_GP_CNTRL_REG_FLAG_SW_RESET); 45 usleep_range(10000, 20000); 46 } else { 47 iwl_set_bit(trans, CSR_RESET, 48 CSR_RESET_REG_FLAG_SW_RESET); 49 usleep_range(5000, 6000); 50 } 51 52 if (retake_ownership) 53 return iwl_pcie_prepare_card_hw(trans); 54 55 return 0; 56 } 57 58 static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans) 59 { 60 struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon; 61 62 if (!fw_mon->size) 63 return; 64 65 dma_free_coherent(trans->dev, fw_mon->size, fw_mon->block, 66 fw_mon->physical); 67 68 fw_mon->block = NULL; 69 fw_mon->physical = 0; 70 fw_mon->size = 0; 71 } 72 73 static void iwl_pcie_alloc_fw_monitor_block(struct iwl_trans *trans, 74 u8 max_power) 75 { 76 struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon; 77 void *block = NULL; 78 dma_addr_t physical = 0; 79 u32 size = 0; 80 u8 power; 81 82 if (fw_mon->size) { 83 memset(fw_mon->block, 0, fw_mon->size); 84 return; 85 } 86 87 /* need at least 2 KiB, so stop at 11 */ 88 for (power = max_power; power >= 11; power--) { 89 size = BIT(power); 90 block = dma_alloc_coherent(trans->dev, size, &physical, 91 GFP_KERNEL | __GFP_NOWARN); 92 if (!block) 93 continue; 94 95 IWL_INFO(trans, 96 "Allocated 0x%08x bytes for firmware monitor.\n", 97 size); 98 break; 99 } 100 101 if (WARN_ON_ONCE(!block)) 102 return; 103 104 if (power != max_power) 105 IWL_ERR(trans, 106 "Sorry - debug buffer is only %luK while you requested %luK\n", 107 (unsigned long)BIT(power - 10), 108 (unsigned long)BIT(max_power - 10)); 109 110 fw_mon->block = block; 111 fw_mon->physical = physical; 112 fw_mon->size = size; 113 } 114 115 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power) 116 { 117 if (!max_power) { 118 /* default max_power is maximum */ 119 max_power = 26; 120 } else { 121 max_power += 11; 122 } 123 124 if (WARN(max_power > 26, 125 "External buffer size for monitor is too big %d, check the FW TLV\n", 126 max_power)) 127 return; 128 129 iwl_pcie_alloc_fw_monitor_block(trans, max_power); 130 } 131 132 static u32 iwl_trans_pcie_read_shr(struct iwl_trans *trans, u32 reg) 133 { 134 iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG, 135 ((reg & 0x0000ffff) | (2 << 28))); 136 return iwl_read32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG); 137 } 138 139 static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val) 140 { 141 iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG, val); 142 iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG, 143 ((reg & 0x0000ffff) | (3 << 28))); 144 } 145 146 static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux) 147 { 148 if (trans->mac_cfg->base->apmg_not_supported) 149 return; 150 151 if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold)) 152 iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG, 153 APMG_PS_CTRL_VAL_PWR_SRC_VAUX, 154 ~APMG_PS_CTRL_MSK_PWR_SRC); 155 else 156 iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG, 157 APMG_PS_CTRL_VAL_PWR_SRC_VMAIN, 158 ~APMG_PS_CTRL_MSK_PWR_SRC); 159 } 160 161 /* PCI registers */ 162 #define PCI_CFG_RETRY_TIMEOUT 0x041 163 164 void iwl_pcie_apm_config(struct iwl_trans *trans) 165 { 166 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 167 u16 lctl; 168 u16 cap; 169 170 /* 171 * L0S states have been found to be unstable with our devices 172 * and in newer hardware they are not officially supported at 173 * all, so we must always set the L0S_DISABLED bit. 174 */ 175 iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_DISABLED); 176 177 pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl); 178 trans->pm_support = !(lctl & PCI_EXP_LNKCTL_ASPM_L0S); 179 180 pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap); 181 trans->ltr_enabled = cap & PCI_EXP_DEVCTL2_LTR_EN; 182 IWL_DEBUG_POWER(trans, "L1 %sabled - LTR %sabled\n", 183 (lctl & PCI_EXP_LNKCTL_ASPM_L1) ? "En" : "Dis", 184 trans->ltr_enabled ? "En" : "Dis"); 185 } 186 187 /* 188 * Start up NIC's basic functionality after it has been reset 189 * (e.g. after platform boot, or shutdown via iwl_pcie_apm_stop()) 190 * NOTE: This does not load uCode nor start the embedded processor 191 */ 192 static int iwl_pcie_apm_init(struct iwl_trans *trans) 193 { 194 int ret; 195 196 IWL_DEBUG_INFO(trans, "Init card's basic functions\n"); 197 198 /* 199 * Use "set_bit" below rather than "write", to preserve any hardware 200 * bits already set by default after reset. 201 */ 202 203 /* Disable L0S exit timer (platform NMI Work/Around) */ 204 if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_8000) 205 iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS, 206 CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER); 207 208 /* 209 * Disable L0s without affecting L1; 210 * don't wait for ICH L0s (ICH bug W/A) 211 */ 212 iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS, 213 CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX); 214 215 /* Set FH wait threshold to maximum (HW error during stress W/A) */ 216 iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL); 217 218 /* 219 * Enable HAP INTA (interrupt from management bus) to 220 * wake device's PCI Express link L1a -> L0s 221 */ 222 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 223 CSR_HW_IF_CONFIG_REG_HAP_WAKE); 224 225 iwl_pcie_apm_config(trans); 226 227 /* Configure analog phase-lock-loop before activating to D0A */ 228 if (trans->mac_cfg->base->pll_cfg) 229 iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL); 230 231 ret = iwl_finish_nic_init(trans); 232 if (ret) 233 return ret; 234 235 if (trans->cfg->host_interrupt_operation_mode) { 236 /* 237 * This is a bit of an abuse - This is needed for 7260 / 3160 238 * only check host_interrupt_operation_mode even if this is 239 * not related to host_interrupt_operation_mode. 240 * 241 * Enable the oscillator to count wake up time for L1 exit. This 242 * consumes slightly more power (100uA) - but allows to be sure 243 * that we wake up from L1 on time. 244 * 245 * This looks weird: read twice the same register, discard the 246 * value, set a bit, and yet again, read that same register 247 * just to discard the value. But that's the way the hardware 248 * seems to like it. 249 */ 250 iwl_read_prph(trans, OSC_CLK); 251 iwl_read_prph(trans, OSC_CLK); 252 iwl_set_bits_prph(trans, OSC_CLK, OSC_CLK_FORCE_CONTROL); 253 iwl_read_prph(trans, OSC_CLK); 254 iwl_read_prph(trans, OSC_CLK); 255 } 256 257 /* 258 * Enable DMA clock and wait for it to stabilize. 259 * 260 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0" 261 * bits do not disable clocks. This preserves any hardware 262 * bits already set by default in "CLK_CTRL_REG" after reset. 263 */ 264 if (!trans->mac_cfg->base->apmg_not_supported) { 265 iwl_write_prph(trans, APMG_CLK_EN_REG, 266 APMG_CLK_VAL_DMA_CLK_RQT); 267 udelay(20); 268 269 /* Disable L1-Active */ 270 iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG, 271 APMG_PCIDEV_STT_VAL_L1_ACT_DIS); 272 273 /* Clear the interrupt in APMG if the NIC is in RFKILL */ 274 iwl_write_prph(trans, APMG_RTC_INT_STT_REG, 275 APMG_RTC_INT_STT_RFKILL); 276 } 277 278 set_bit(STATUS_DEVICE_ENABLED, &trans->status); 279 280 return 0; 281 } 282 283 /* 284 * Enable LP XTAL to avoid HW bug where device may consume much power if 285 * FW is not loaded after device reset. LP XTAL is disabled by default 286 * after device HW reset. Do it only if XTAL is fed by internal source. 287 * Configure device's "persistence" mode to avoid resetting XTAL again when 288 * SHRD_HW_RST occurs in S3. 289 */ 290 static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans) 291 { 292 int ret; 293 u32 apmg_gp1_reg; 294 u32 apmg_xtal_cfg_reg; 295 u32 dl_cfg_reg; 296 297 /* Force XTAL ON */ 298 iwl_trans_set_bit(trans, CSR_GP_CNTRL, 299 CSR_GP_CNTRL_REG_FLAG_XTAL_ON); 300 301 ret = iwl_trans_pcie_sw_reset(trans, true); 302 303 if (!ret) 304 ret = iwl_finish_nic_init(trans); 305 306 if (WARN_ON(ret)) { 307 /* Release XTAL ON request */ 308 iwl_trans_clear_bit(trans, CSR_GP_CNTRL, 309 CSR_GP_CNTRL_REG_FLAG_XTAL_ON); 310 return; 311 } 312 313 /* 314 * Clear "disable persistence" to avoid LP XTAL resetting when 315 * SHRD_HW_RST is applied in S3. 316 */ 317 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG, 318 APMG_PCIDEV_STT_VAL_PERSIST_DIS); 319 320 /* 321 * Force APMG XTAL to be active to prevent its disabling by HW 322 * caused by APMG idle state. 323 */ 324 apmg_xtal_cfg_reg = iwl_trans_pcie_read_shr(trans, 325 SHR_APMG_XTAL_CFG_REG); 326 iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG, 327 apmg_xtal_cfg_reg | 328 SHR_APMG_XTAL_CFG_XTAL_ON_REQ); 329 330 ret = iwl_trans_pcie_sw_reset(trans, true); 331 if (ret) 332 IWL_ERR(trans, 333 "iwl_pcie_apm_lp_xtal_enable: failed to retake NIC ownership\n"); 334 335 /* Enable LP XTAL by indirect access through CSR */ 336 apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG); 337 iwl_trans_pcie_write_shr(trans, SHR_APMG_GP1_REG, apmg_gp1_reg | 338 SHR_APMG_GP1_WF_XTAL_LP_EN | 339 SHR_APMG_GP1_CHICKEN_BIT_SELECT); 340 341 /* Clear delay line clock power up */ 342 dl_cfg_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_DL_CFG_REG); 343 iwl_trans_pcie_write_shr(trans, SHR_APMG_DL_CFG_REG, dl_cfg_reg & 344 ~SHR_APMG_DL_CFG_DL_CLOCK_POWER_UP); 345 346 /* 347 * Enable persistence mode to avoid LP XTAL resetting when 348 * SHRD_HW_RST is applied in S3. 349 */ 350 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 351 CSR_HW_IF_CONFIG_REG_PERSISTENCE); 352 353 /* 354 * Clear "initialization complete" bit to move adapter from 355 * D0A* (powered-up Active) --> D0U* (Uninitialized) state. 356 */ 357 iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE); 358 359 /* Activates XTAL resources monitor */ 360 iwl_trans_set_bit(trans, CSR_MONITOR_CFG_REG, 361 CSR_MONITOR_XTAL_RESOURCES); 362 363 /* Release XTAL ON request */ 364 iwl_trans_clear_bit(trans, CSR_GP_CNTRL, 365 CSR_GP_CNTRL_REG_FLAG_XTAL_ON); 366 udelay(10); 367 368 /* Release APMG XTAL */ 369 iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG, 370 apmg_xtal_cfg_reg & 371 ~SHR_APMG_XTAL_CFG_XTAL_ON_REQ); 372 } 373 374 void iwl_pcie_apm_stop_master(struct iwl_trans *trans) 375 { 376 int ret; 377 378 /* stop device's busmaster DMA activity */ 379 380 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { 381 iwl_set_bit(trans, CSR_GP_CNTRL, 382 CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_REQ); 383 384 ret = iwl_poll_bits(trans, CSR_GP_CNTRL, 385 CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS, 386 100); 387 usleep_range(10000, 20000); 388 } else { 389 iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER); 390 391 ret = iwl_poll_bits(trans, CSR_RESET, 392 CSR_RESET_REG_FLAG_MASTER_DISABLED, 100); 393 } 394 395 if (ret < 0) 396 IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n"); 397 398 IWL_DEBUG_INFO(trans, "stop master\n"); 399 } 400 401 static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave) 402 { 403 IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n"); 404 405 if (op_mode_leave) { 406 if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status)) 407 iwl_pcie_apm_init(trans); 408 409 /* inform ME that we are leaving */ 410 if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000) 411 iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG, 412 APMG_PCIDEV_STT_VAL_WAKE_ME); 413 else if (trans->mac_cfg->device_family >= 414 IWL_DEVICE_FAMILY_8000) { 415 iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG, 416 CSR_RESET_LINK_PWR_MGMT_DISABLED); 417 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 418 CSR_HW_IF_CONFIG_REG_WAKE_ME | 419 CSR_HW_IF_CONFIG_REG_WAKE_ME_PCIE_OWNER_EN); 420 mdelay(1); 421 iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG, 422 CSR_RESET_LINK_PWR_MGMT_DISABLED); 423 } 424 mdelay(5); 425 } 426 427 clear_bit(STATUS_DEVICE_ENABLED, &trans->status); 428 429 /* Stop device's DMA activity */ 430 iwl_pcie_apm_stop_master(trans); 431 432 if (trans->cfg->lp_xtal_workaround) { 433 iwl_pcie_apm_lp_xtal_enable(trans); 434 return; 435 } 436 437 iwl_trans_pcie_sw_reset(trans, false); 438 439 /* 440 * Clear "initialization complete" bit to move adapter from 441 * D0A* (powered-up Active) --> D0U* (Uninitialized) state. 442 */ 443 iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE); 444 } 445 446 static int iwl_pcie_nic_init(struct iwl_trans *trans) 447 { 448 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 449 int ret; 450 451 /* nic_init */ 452 spin_lock_bh(&trans_pcie->irq_lock); 453 ret = iwl_pcie_apm_init(trans); 454 spin_unlock_bh(&trans_pcie->irq_lock); 455 456 if (ret) 457 return ret; 458 459 iwl_pcie_set_pwr(trans, false); 460 461 iwl_op_mode_nic_config(trans->op_mode); 462 463 /* Allocate the RX queue, or reset if it is already allocated */ 464 ret = iwl_pcie_rx_init(trans); 465 if (ret) 466 return ret; 467 468 /* Allocate or reset and init all Tx and Command queues */ 469 if (iwl_pcie_tx_init(trans)) { 470 iwl_pcie_rx_free(trans); 471 return -ENOMEM; 472 } 473 474 if (trans->mac_cfg->base->shadow_reg_enable) { 475 /* enable shadow regs in HW */ 476 iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF); 477 IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n"); 478 } 479 480 return 0; 481 } 482 483 #define HW_READY_TIMEOUT (50) 484 485 /* Note: returns poll_bit return value, which is >= 0 if success */ 486 static int iwl_pcie_set_hw_ready(struct iwl_trans *trans) 487 { 488 int ret; 489 490 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 491 CSR_HW_IF_CONFIG_REG_PCI_OWN_SET); 492 493 /* See if we got it */ 494 ret = iwl_poll_bits(trans, CSR_HW_IF_CONFIG_REG, 495 CSR_HW_IF_CONFIG_REG_PCI_OWN_SET, 496 HW_READY_TIMEOUT); 497 498 if (ret >= 0) 499 iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE); 500 501 IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : ""); 502 return ret; 503 } 504 505 /* Note: returns standard 0/-ERROR code */ 506 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans) 507 { 508 int ret; 509 int iter; 510 511 IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n"); 512 513 ret = iwl_pcie_set_hw_ready(trans); 514 /* If the card is ready, exit 0 */ 515 if (ret >= 0) { 516 trans->csme_own = false; 517 return 0; 518 } 519 520 iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG, 521 CSR_RESET_LINK_PWR_MGMT_DISABLED); 522 usleep_range(1000, 2000); 523 524 for (iter = 0; iter < 10; iter++) { 525 int t = 0; 526 527 /* If HW is not ready, prepare the conditions to check again */ 528 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 529 CSR_HW_IF_CONFIG_REG_WAKE_ME); 530 531 do { 532 ret = iwl_pcie_set_hw_ready(trans); 533 if (ret >= 0) { 534 trans->csme_own = false; 535 return 0; 536 } 537 538 if (iwl_mei_is_connected()) { 539 IWL_DEBUG_INFO(trans, 540 "Couldn't prepare the card but SAP is connected\n"); 541 trans->csme_own = true; 542 if (trans->mac_cfg->device_family != 543 IWL_DEVICE_FAMILY_9000) 544 IWL_ERR(trans, 545 "SAP not supported for this NIC family\n"); 546 547 return -EBUSY; 548 } 549 550 usleep_range(200, 1000); 551 t += 200; 552 } while (t < 150000); 553 msleep(25); 554 } 555 556 IWL_ERR(trans, "Couldn't prepare the card\n"); 557 558 return ret; 559 } 560 561 /* 562 * ucode 563 */ 564 static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans, 565 u32 dst_addr, dma_addr_t phy_addr, 566 u32 byte_cnt) 567 { 568 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL), 569 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE); 570 571 iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL), 572 dst_addr); 573 574 iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL), 575 phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK); 576 577 iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL), 578 (iwl_get_dma_hi_addr(phy_addr) 579 << FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt); 580 581 iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL), 582 BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) | 583 BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) | 584 FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID); 585 586 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL), 587 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | 588 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE | 589 FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD); 590 } 591 592 static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans, 593 u32 dst_addr, dma_addr_t phy_addr, 594 u32 byte_cnt) 595 { 596 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 597 int ret; 598 599 trans_pcie->ucode_write_complete = false; 600 601 if (!iwl_trans_grab_nic_access(trans)) 602 return -EIO; 603 604 iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr, 605 byte_cnt); 606 iwl_trans_release_nic_access(trans); 607 608 ret = wait_event_timeout(trans_pcie->ucode_write_waitq, 609 trans_pcie->ucode_write_complete, 5 * HZ); 610 if (!ret) { 611 IWL_ERR(trans, "Failed to load firmware chunk!\n"); 612 iwl_trans_pcie_dump_regs(trans, trans_pcie->pci_dev); 613 return -ETIMEDOUT; 614 } 615 616 return 0; 617 } 618 619 static int iwl_pcie_load_section(struct iwl_trans *trans, u8 section_num, 620 const struct fw_desc *section) 621 { 622 u8 *v_addr; 623 dma_addr_t p_addr; 624 u32 offset, chunk_sz = min_t(u32, FH_MEM_TB_MAX_LENGTH, section->len); 625 int ret = 0; 626 627 IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n", 628 section_num); 629 630 v_addr = dma_alloc_coherent(trans->dev, chunk_sz, &p_addr, 631 GFP_KERNEL | __GFP_NOWARN); 632 if (!v_addr) { 633 IWL_DEBUG_INFO(trans, "Falling back to small chunks of DMA\n"); 634 chunk_sz = PAGE_SIZE; 635 v_addr = dma_alloc_coherent(trans->dev, chunk_sz, 636 &p_addr, GFP_KERNEL); 637 if (!v_addr) 638 return -ENOMEM; 639 } 640 641 for (offset = 0; offset < section->len; offset += chunk_sz) { 642 u32 copy_size, dst_addr; 643 bool extended_addr = false; 644 645 copy_size = min_t(u32, chunk_sz, section->len - offset); 646 dst_addr = section->offset + offset; 647 648 if (dst_addr >= IWL_FW_MEM_EXTENDED_START && 649 dst_addr <= IWL_FW_MEM_EXTENDED_END) 650 extended_addr = true; 651 652 if (extended_addr) 653 iwl_set_bits_prph(trans, LMPM_CHICK, 654 LMPM_CHICK_EXTENDED_ADDR_SPACE); 655 656 memcpy(v_addr, (const u8 *)section->data + offset, copy_size); 657 ret = iwl_pcie_load_firmware_chunk(trans, dst_addr, p_addr, 658 copy_size); 659 660 if (extended_addr) 661 iwl_clear_bits_prph(trans, LMPM_CHICK, 662 LMPM_CHICK_EXTENDED_ADDR_SPACE); 663 664 if (ret) { 665 IWL_ERR(trans, 666 "Could not load the [%d] uCode section\n", 667 section_num); 668 break; 669 } 670 } 671 672 dma_free_coherent(trans->dev, chunk_sz, v_addr, p_addr); 673 return ret; 674 } 675 676 static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans, 677 const struct fw_img *image, 678 int cpu, 679 int *first_ucode_section) 680 { 681 int shift_param; 682 int i, ret = 0, sec_num = 0x1; 683 u32 val, last_read_idx = 0; 684 685 if (cpu == 1) { 686 shift_param = 0; 687 *first_ucode_section = 0; 688 } else { 689 shift_param = 16; 690 (*first_ucode_section)++; 691 } 692 693 for (i = *first_ucode_section; i < image->num_sec; i++) { 694 last_read_idx = i; 695 696 /* 697 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between 698 * CPU1 to CPU2. 699 * PAGING_SEPARATOR_SECTION delimiter - separate between 700 * CPU2 non paged to CPU2 paging sec. 701 */ 702 if (!image->sec[i].data || 703 image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION || 704 image->sec[i].offset == PAGING_SEPARATOR_SECTION) { 705 IWL_DEBUG_FW(trans, 706 "Break since Data not valid or Empty section, sec = %d\n", 707 i); 708 break; 709 } 710 711 ret = iwl_pcie_load_section(trans, i, &image->sec[i]); 712 if (ret) 713 return ret; 714 715 /* Notify ucode of loaded section number and status */ 716 val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS); 717 val = val | (sec_num << shift_param); 718 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val); 719 720 sec_num = (sec_num << 1) | 0x1; 721 } 722 723 *first_ucode_section = last_read_idx; 724 725 iwl_enable_interrupts(trans); 726 727 if (trans->mac_cfg->gen2) { 728 if (cpu == 1) 729 iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS, 730 0xFFFF); 731 else 732 iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS, 733 0xFFFFFFFF); 734 } else { 735 if (cpu == 1) 736 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, 737 0xFFFF); 738 else 739 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, 740 0xFFFFFFFF); 741 } 742 743 return 0; 744 } 745 746 static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans, 747 const struct fw_img *image, 748 int cpu, 749 int *first_ucode_section) 750 { 751 int i, ret = 0; 752 u32 last_read_idx = 0; 753 754 if (cpu == 1) 755 *first_ucode_section = 0; 756 else 757 (*first_ucode_section)++; 758 759 for (i = *first_ucode_section; i < image->num_sec; i++) { 760 last_read_idx = i; 761 762 /* 763 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between 764 * CPU1 to CPU2. 765 * PAGING_SEPARATOR_SECTION delimiter - separate between 766 * CPU2 non paged to CPU2 paging sec. 767 */ 768 if (!image->sec[i].data || 769 image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION || 770 image->sec[i].offset == PAGING_SEPARATOR_SECTION) { 771 IWL_DEBUG_FW(trans, 772 "Break since Data not valid or Empty section, sec = %d\n", 773 i); 774 break; 775 } 776 777 ret = iwl_pcie_load_section(trans, i, &image->sec[i]); 778 if (ret) 779 return ret; 780 } 781 782 *first_ucode_section = last_read_idx; 783 784 return 0; 785 } 786 787 static void iwl_pcie_apply_destination_ini(struct iwl_trans *trans) 788 { 789 enum iwl_fw_ini_allocation_id alloc_id = IWL_FW_INI_ALLOCATION_ID_DBGC1; 790 struct iwl_fw_ini_allocation_tlv *fw_mon_cfg = 791 &trans->dbg.fw_mon_cfg[alloc_id]; 792 struct iwl_dram_data *frag; 793 794 if (!iwl_trans_dbg_ini_valid(trans)) 795 return; 796 797 if (le32_to_cpu(fw_mon_cfg->buf_location) == 798 IWL_FW_INI_LOCATION_SRAM_PATH) { 799 IWL_DEBUG_FW(trans, "WRT: Applying SMEM buffer destination\n"); 800 /* set sram monitor by enabling bit 7 */ 801 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 802 CSR_HW_IF_CONFIG_REG_BIT_MONITOR_SRAM); 803 804 return; 805 } 806 807 if (le32_to_cpu(fw_mon_cfg->buf_location) != 808 IWL_FW_INI_LOCATION_DRAM_PATH || 809 !trans->dbg.fw_mon_ini[alloc_id].num_frags) 810 return; 811 812 frag = &trans->dbg.fw_mon_ini[alloc_id].frags[0]; 813 814 IWL_DEBUG_FW(trans, "WRT: Applying DRAM destination (alloc_id=%u)\n", 815 alloc_id); 816 817 iwl_write_umac_prph(trans, MON_BUFF_BASE_ADDR_VER2, 818 frag->physical >> MON_BUFF_SHIFT_VER2); 819 iwl_write_umac_prph(trans, MON_BUFF_END_ADDR_VER2, 820 (frag->physical + frag->size - 256) >> 821 MON_BUFF_SHIFT_VER2); 822 } 823 824 void iwl_pcie_apply_destination(struct iwl_trans *trans) 825 { 826 const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg.dest_tlv; 827 const struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon; 828 int i; 829 830 if (iwl_trans_dbg_ini_valid(trans)) { 831 iwl_pcie_apply_destination_ini(trans); 832 return; 833 } 834 835 IWL_INFO(trans, "Applying debug destination %s\n", 836 get_fw_dbg_mode_string(dest->monitor_mode)); 837 838 if (dest->monitor_mode == EXTERNAL_MODE) 839 iwl_pcie_alloc_fw_monitor(trans, dest->size_power); 840 else 841 IWL_WARN(trans, "PCI should have external buffer debug\n"); 842 843 for (i = 0; i < trans->dbg.n_dest_reg; i++) { 844 u32 addr = le32_to_cpu(dest->reg_ops[i].addr); 845 u32 val = le32_to_cpu(dest->reg_ops[i].val); 846 847 switch (dest->reg_ops[i].op) { 848 case CSR_ASSIGN: 849 iwl_write32(trans, addr, val); 850 break; 851 case CSR_SETBIT: 852 iwl_set_bit(trans, addr, BIT(val)); 853 break; 854 case CSR_CLEARBIT: 855 iwl_clear_bit(trans, addr, BIT(val)); 856 break; 857 case PRPH_ASSIGN: 858 iwl_write_prph(trans, addr, val); 859 break; 860 case PRPH_SETBIT: 861 iwl_set_bits_prph(trans, addr, BIT(val)); 862 break; 863 case PRPH_CLEARBIT: 864 iwl_clear_bits_prph(trans, addr, BIT(val)); 865 break; 866 case PRPH_BLOCKBIT: 867 if (iwl_read_prph(trans, addr) & BIT(val)) { 868 IWL_ERR(trans, 869 "BIT(%u) in address 0x%x is 1, stopping FW configuration\n", 870 val, addr); 871 goto monitor; 872 } 873 break; 874 default: 875 IWL_ERR(trans, "FW debug - unknown OP %d\n", 876 dest->reg_ops[i].op); 877 break; 878 } 879 } 880 881 monitor: 882 if (dest->monitor_mode == EXTERNAL_MODE && fw_mon->size) { 883 iwl_write_prph(trans, le32_to_cpu(dest->base_reg), 884 fw_mon->physical >> dest->base_shift); 885 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000) 886 iwl_write_prph(trans, le32_to_cpu(dest->end_reg), 887 (fw_mon->physical + fw_mon->size - 888 256) >> dest->end_shift); 889 else 890 iwl_write_prph(trans, le32_to_cpu(dest->end_reg), 891 (fw_mon->physical + fw_mon->size) >> 892 dest->end_shift); 893 } 894 } 895 896 static int iwl_pcie_load_given_ucode(struct iwl_trans *trans, 897 const struct fw_img *image) 898 { 899 int ret = 0; 900 int first_ucode_section; 901 902 IWL_DEBUG_FW(trans, "working with %s CPU\n", 903 image->is_dual_cpus ? "Dual" : "Single"); 904 905 /* load to FW the binary non secured sections of CPU1 */ 906 ret = iwl_pcie_load_cpu_sections(trans, image, 1, &first_ucode_section); 907 if (ret) 908 return ret; 909 910 if (image->is_dual_cpus) { 911 /* set CPU2 header address */ 912 iwl_write_prph(trans, 913 LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR, 914 LMPM_SECURE_CPU2_HDR_MEM_SPACE); 915 916 /* load to FW the binary sections of CPU2 */ 917 ret = iwl_pcie_load_cpu_sections(trans, image, 2, 918 &first_ucode_section); 919 if (ret) 920 return ret; 921 } 922 923 if (iwl_pcie_dbg_on(trans)) 924 iwl_pcie_apply_destination(trans); 925 926 iwl_enable_interrupts(trans); 927 928 /* release CPU reset */ 929 iwl_write32(trans, CSR_RESET, 0); 930 931 return 0; 932 } 933 934 static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans, 935 const struct fw_img *image) 936 { 937 int ret = 0; 938 int first_ucode_section; 939 940 IWL_DEBUG_FW(trans, "working with %s CPU\n", 941 image->is_dual_cpus ? "Dual" : "Single"); 942 943 if (iwl_pcie_dbg_on(trans)) 944 iwl_pcie_apply_destination(trans); 945 946 IWL_DEBUG_POWER(trans, "Original WFPM value = 0x%08X\n", 947 iwl_read_prph(trans, WFPM_GP2)); 948 949 /* 950 * Set default value. On resume reading the values that were 951 * zeored can provide debug data on the resume flow. 952 * This is for debugging only and has no functional impact. 953 */ 954 iwl_write_prph(trans, WFPM_GP2, 0x01010101); 955 956 /* configure the ucode to be ready to get the secured image */ 957 /* release CPU reset */ 958 iwl_write_prph(trans, RELEASE_CPU_RESET, RELEASE_CPU_RESET_BIT); 959 960 /* load to FW the binary Secured sections of CPU1 */ 961 ret = iwl_pcie_load_cpu_sections_8000(trans, image, 1, 962 &first_ucode_section); 963 if (ret) 964 return ret; 965 966 /* load to FW the binary sections of CPU2 */ 967 return iwl_pcie_load_cpu_sections_8000(trans, image, 2, 968 &first_ucode_section); 969 } 970 971 bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans) 972 { 973 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 974 bool hw_rfkill = iwl_is_rfkill_set(trans); 975 bool prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status); 976 bool report; 977 978 if (hw_rfkill) { 979 set_bit(STATUS_RFKILL_HW, &trans->status); 980 set_bit(STATUS_RFKILL_OPMODE, &trans->status); 981 } else { 982 clear_bit(STATUS_RFKILL_HW, &trans->status); 983 if (trans_pcie->opmode_down) 984 clear_bit(STATUS_RFKILL_OPMODE, &trans->status); 985 } 986 987 report = test_bit(STATUS_RFKILL_OPMODE, &trans->status); 988 989 if (prev != report) 990 iwl_trans_pcie_rf_kill(trans, report, false); 991 992 return hw_rfkill; 993 } 994 995 struct iwl_causes_list { 996 u16 mask_reg; 997 u8 bit; 998 u8 addr; 999 }; 1000 1001 #define IWL_CAUSE(reg, mask) \ 1002 { \ 1003 .mask_reg = reg, \ 1004 .bit = ilog2(mask), \ 1005 .addr = ilog2(mask) + \ 1006 ((reg) == CSR_MSIX_FH_INT_MASK_AD ? -16 : \ 1007 (reg) == CSR_MSIX_HW_INT_MASK_AD ? 16 : \ 1008 0xffff), /* causes overflow warning */ \ 1009 } 1010 1011 static const struct iwl_causes_list causes_list_common[] = { 1012 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_D2S_CH0_NUM), 1013 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_D2S_CH1_NUM), 1014 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_S2D), 1015 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_FH_ERR), 1016 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_ALIVE), 1017 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_WAKEUP), 1018 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_RESET_DONE), 1019 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_TOP_FATAL_ERR), 1020 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_CT_KILL), 1021 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_RF_KILL), 1022 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_PERIODIC), 1023 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SCD), 1024 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_FH_TX), 1025 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_HW_ERR), 1026 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_HAP), 1027 }; 1028 1029 static const struct iwl_causes_list causes_list_pre_bz[] = { 1030 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SW_ERR), 1031 }; 1032 1033 static const struct iwl_causes_list causes_list_bz[] = { 1034 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ), 1035 }; 1036 1037 static void iwl_pcie_map_list(struct iwl_trans *trans, 1038 const struct iwl_causes_list *causes, 1039 int arr_size, int val) 1040 { 1041 int i; 1042 1043 for (i = 0; i < arr_size; i++) { 1044 iwl_write8(trans, CSR_MSIX_IVAR(causes[i].addr), val); 1045 iwl_clear_bit(trans, causes[i].mask_reg, 1046 BIT(causes[i].bit)); 1047 } 1048 } 1049 1050 static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans) 1051 { 1052 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1053 int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE; 1054 /* 1055 * Access all non RX causes and map them to the default irq. 1056 * In case we are missing at least one interrupt vector, 1057 * the first interrupt vector will serve non-RX and FBQ causes. 1058 */ 1059 iwl_pcie_map_list(trans, causes_list_common, 1060 ARRAY_SIZE(causes_list_common), val); 1061 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) 1062 iwl_pcie_map_list(trans, causes_list_bz, 1063 ARRAY_SIZE(causes_list_bz), val); 1064 else 1065 iwl_pcie_map_list(trans, causes_list_pre_bz, 1066 ARRAY_SIZE(causes_list_pre_bz), val); 1067 } 1068 1069 static void iwl_pcie_map_rx_causes(struct iwl_trans *trans) 1070 { 1071 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1072 u32 offset = 1073 trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0; 1074 u32 val, idx; 1075 1076 /* 1077 * The first RX queue - fallback queue, which is designated for 1078 * management frame, command responses etc, is always mapped to the 1079 * first interrupt vector. The other RX queues are mapped to 1080 * the other (N - 2) interrupt vectors. 1081 */ 1082 val = BIT(MSIX_FH_INT_CAUSES_Q(0)); 1083 for (idx = 1; idx < trans->info.num_rxqs; idx++) { 1084 iwl_write8(trans, CSR_MSIX_RX_IVAR(idx), 1085 MSIX_FH_INT_CAUSES_Q(idx - offset)); 1086 val |= BIT(MSIX_FH_INT_CAUSES_Q(idx)); 1087 } 1088 iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val); 1089 1090 val = MSIX_FH_INT_CAUSES_Q(0); 1091 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) 1092 val |= MSIX_NON_AUTO_CLEAR_CAUSE; 1093 iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val); 1094 1095 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) 1096 iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val); 1097 } 1098 1099 void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie) 1100 { 1101 struct iwl_trans *trans = trans_pcie->trans; 1102 1103 if (!trans_pcie->msix_enabled) { 1104 if (trans->mac_cfg->mq_rx_supported && 1105 test_bit(STATUS_DEVICE_ENABLED, &trans->status)) 1106 iwl_write_umac_prph(trans, UREG_CHICK, 1107 UREG_CHICK_MSI_ENABLE); 1108 return; 1109 } 1110 /* 1111 * The IVAR table needs to be configured again after reset, 1112 * but if the device is disabled, we can't write to 1113 * prph. 1114 */ 1115 if (test_bit(STATUS_DEVICE_ENABLED, &trans->status)) 1116 iwl_write_umac_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE); 1117 1118 /* 1119 * Each cause from the causes list above and the RX causes is 1120 * represented as a byte in the IVAR table. The first nibble 1121 * represents the bound interrupt vector of the cause, the second 1122 * represents no auto clear for this cause. This will be set if its 1123 * interrupt vector is bound to serve other causes. 1124 */ 1125 iwl_pcie_map_rx_causes(trans); 1126 1127 iwl_pcie_map_non_rx_causes(trans); 1128 } 1129 1130 static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie) 1131 { 1132 struct iwl_trans *trans = trans_pcie->trans; 1133 1134 iwl_pcie_conf_msix_hw(trans_pcie); 1135 1136 if (!trans_pcie->msix_enabled) 1137 return; 1138 1139 trans_pcie->fh_init_mask = ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD); 1140 trans_pcie->fh_mask = trans_pcie->fh_init_mask; 1141 trans_pcie->hw_init_mask = ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD); 1142 trans_pcie->hw_mask = trans_pcie->hw_init_mask; 1143 } 1144 1145 static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool from_irq) 1146 { 1147 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1148 1149 lockdep_assert_held(&trans_pcie->mutex); 1150 1151 if (trans_pcie->is_down) 1152 return; 1153 1154 trans_pcie->is_down = true; 1155 1156 /* tell the device to stop sending interrupts */ 1157 iwl_disable_interrupts(trans); 1158 1159 /* device going down, Stop using ICT table */ 1160 iwl_pcie_disable_ict(trans); 1161 1162 /* 1163 * If a HW restart happens during firmware loading, 1164 * then the firmware loading might call this function 1165 * and later it might be called again due to the 1166 * restart. So don't process again if the device is 1167 * already dead. 1168 */ 1169 if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) { 1170 IWL_DEBUG_INFO(trans, 1171 "DEVICE_ENABLED bit was set and is now cleared\n"); 1172 if (!from_irq) 1173 iwl_pcie_synchronize_irqs(trans); 1174 iwl_pcie_rx_napi_sync(trans); 1175 iwl_pcie_tx_stop(trans); 1176 iwl_pcie_rx_stop(trans); 1177 1178 /* Power-down device's busmaster DMA clocks */ 1179 if (!trans->mac_cfg->base->apmg_not_supported) { 1180 iwl_write_prph(trans, APMG_CLK_DIS_REG, 1181 APMG_CLK_VAL_DMA_CLK_RQT); 1182 udelay(5); 1183 } 1184 } 1185 1186 /* Make sure (redundant) we've released our request to stay awake */ 1187 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) 1188 iwl_clear_bit(trans, CSR_GP_CNTRL, 1189 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ); 1190 else 1191 iwl_clear_bit(trans, CSR_GP_CNTRL, 1192 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 1193 1194 /* Stop the device, and put it in low power state */ 1195 iwl_pcie_apm_stop(trans, false); 1196 1197 /* re-take ownership to prevent other users from stealing the device */ 1198 iwl_trans_pcie_sw_reset(trans, true); 1199 1200 /* 1201 * Upon stop, the IVAR table gets erased, so msi-x won't 1202 * work. This causes a bug in RF-KILL flows, since the interrupt 1203 * that enables radio won't fire on the correct irq, and the 1204 * driver won't be able to handle the interrupt. 1205 * Configure the IVAR table again after reset. 1206 */ 1207 iwl_pcie_conf_msix_hw(trans_pcie); 1208 1209 /* 1210 * Upon stop, the APM issues an interrupt if HW RF kill is set. 1211 * This is a bug in certain verions of the hardware. 1212 * Certain devices also keep sending HW RF kill interrupt all 1213 * the time, unless the interrupt is ACKed even if the interrupt 1214 * should be masked. Re-ACK all the interrupts here. 1215 */ 1216 iwl_disable_interrupts(trans); 1217 1218 /* clear all status bits */ 1219 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); 1220 clear_bit(STATUS_INT_ENABLED, &trans->status); 1221 clear_bit(STATUS_TPOWER_PMI, &trans->status); 1222 1223 /* 1224 * Even if we stop the HW, we still want the RF kill 1225 * interrupt 1226 */ 1227 iwl_enable_rfkill_int(trans); 1228 } 1229 1230 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans) 1231 { 1232 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1233 1234 if (trans_pcie->msix_enabled) { 1235 int i; 1236 1237 for (i = 0; i < trans_pcie->alloc_vecs; i++) 1238 synchronize_irq(trans_pcie->msix_entries[i].vector); 1239 } else { 1240 synchronize_irq(trans_pcie->pci_dev->irq); 1241 } 1242 } 1243 1244 int iwl_trans_pcie_start_fw(struct iwl_trans *trans, 1245 const struct iwl_fw *fw, 1246 const struct fw_img *img, 1247 bool run_in_rfkill) 1248 { 1249 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1250 bool hw_rfkill; 1251 int ret; 1252 1253 /* This may fail if AMT took ownership of the device */ 1254 if (iwl_pcie_prepare_card_hw(trans)) { 1255 IWL_WARN(trans, "Exit HW not ready\n"); 1256 return -EIO; 1257 } 1258 1259 iwl_enable_rfkill_int(trans); 1260 1261 iwl_write32(trans, CSR_INT, 0xFFFFFFFF); 1262 1263 /* 1264 * We enabled the RF-Kill interrupt and the handler may very 1265 * well be running. Disable the interrupts to make sure no other 1266 * interrupt can be fired. 1267 */ 1268 iwl_disable_interrupts(trans); 1269 1270 /* Make sure it finished running */ 1271 iwl_pcie_synchronize_irqs(trans); 1272 1273 mutex_lock(&trans_pcie->mutex); 1274 1275 /* If platform's RF_KILL switch is NOT set to KILL */ 1276 hw_rfkill = iwl_pcie_check_hw_rf_kill(trans); 1277 if (hw_rfkill && !run_in_rfkill) { 1278 ret = -ERFKILL; 1279 goto out; 1280 } 1281 1282 /* Someone called stop_device, don't try to start_fw */ 1283 if (trans_pcie->is_down) { 1284 IWL_WARN(trans, 1285 "Can't start_fw since the HW hasn't been started\n"); 1286 ret = -EIO; 1287 goto out; 1288 } 1289 1290 /* make sure rfkill handshake bits are cleared */ 1291 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL); 1292 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, 1293 CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED); 1294 1295 /* clear (again), then enable host interrupts */ 1296 iwl_write32(trans, CSR_INT, 0xFFFFFFFF); 1297 1298 ret = iwl_pcie_nic_init(trans); 1299 if (ret) { 1300 IWL_ERR(trans, "Unable to init nic\n"); 1301 goto out; 1302 } 1303 1304 /* 1305 * Now, we load the firmware and don't want to be interrupted, even 1306 * by the RF-Kill interrupt (hence mask all the interrupt besides the 1307 * FH_TX interrupt which is needed to load the firmware). If the 1308 * RF-Kill switch is toggled, we will find out after having loaded 1309 * the firmware and return the proper value to the caller. 1310 */ 1311 iwl_enable_fw_load_int(trans); 1312 1313 /* really make sure rfkill handshake bits are cleared */ 1314 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL); 1315 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL); 1316 1317 /* Load the given image to the HW */ 1318 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000) 1319 ret = iwl_pcie_load_given_ucode_8000(trans, img); 1320 else 1321 ret = iwl_pcie_load_given_ucode(trans, img); 1322 1323 /* re-check RF-Kill state since we may have missed the interrupt */ 1324 hw_rfkill = iwl_pcie_check_hw_rf_kill(trans); 1325 if (hw_rfkill && !run_in_rfkill) 1326 ret = -ERFKILL; 1327 1328 out: 1329 mutex_unlock(&trans_pcie->mutex); 1330 return ret; 1331 } 1332 1333 void iwl_trans_pcie_fw_alive(struct iwl_trans *trans) 1334 { 1335 iwl_pcie_reset_ict(trans); 1336 iwl_pcie_tx_start(trans); 1337 } 1338 1339 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans, 1340 bool was_in_rfkill) 1341 { 1342 bool hw_rfkill; 1343 1344 /* 1345 * Check again since the RF kill state may have changed while 1346 * all the interrupts were disabled, in this case we couldn't 1347 * receive the RF kill interrupt and update the state in the 1348 * op_mode. 1349 * Don't call the op_mode if the rkfill state hasn't changed. 1350 * This allows the op_mode to call stop_device from the rfkill 1351 * notification without endless recursion. Under very rare 1352 * circumstances, we might have a small recursion if the rfkill 1353 * state changed exactly now while we were called from stop_device. 1354 * This is very unlikely but can happen and is supported. 1355 */ 1356 hw_rfkill = iwl_is_rfkill_set(trans); 1357 if (hw_rfkill) { 1358 set_bit(STATUS_RFKILL_HW, &trans->status); 1359 set_bit(STATUS_RFKILL_OPMODE, &trans->status); 1360 } else { 1361 clear_bit(STATUS_RFKILL_HW, &trans->status); 1362 clear_bit(STATUS_RFKILL_OPMODE, &trans->status); 1363 } 1364 if (hw_rfkill != was_in_rfkill) 1365 iwl_trans_pcie_rf_kill(trans, hw_rfkill, false); 1366 } 1367 1368 void iwl_trans_pcie_stop_device(struct iwl_trans *trans) 1369 { 1370 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1371 bool was_in_rfkill; 1372 1373 iwl_op_mode_time_point(trans->op_mode, 1374 IWL_FW_INI_TIME_POINT_HOST_DEVICE_DISABLE, 1375 NULL); 1376 1377 mutex_lock(&trans_pcie->mutex); 1378 trans_pcie->opmode_down = true; 1379 was_in_rfkill = test_bit(STATUS_RFKILL_OPMODE, &trans->status); 1380 _iwl_trans_pcie_stop_device(trans, false); 1381 iwl_trans_pcie_handle_stop_rfkill(trans, was_in_rfkill); 1382 mutex_unlock(&trans_pcie->mutex); 1383 } 1384 1385 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq) 1386 { 1387 struct iwl_trans_pcie __maybe_unused *trans_pcie = 1388 IWL_TRANS_GET_PCIE_TRANS(trans); 1389 1390 lockdep_assert_held(&trans_pcie->mutex); 1391 1392 IWL_WARN(trans, "reporting RF_KILL (radio %s)\n", 1393 state ? "disabled" : "enabled"); 1394 if (iwl_op_mode_hw_rf_kill(trans->op_mode, state) && 1395 !WARN_ON(trans->mac_cfg->gen2)) 1396 _iwl_trans_pcie_stop_device(trans, from_irq); 1397 } 1398 1399 static void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans, 1400 bool test, bool reset) 1401 { 1402 iwl_disable_interrupts(trans); 1403 1404 /* 1405 * in testing mode, the host stays awake and the 1406 * hardware won't be reset (not even partially) 1407 */ 1408 if (test) 1409 return; 1410 1411 iwl_pcie_disable_ict(trans); 1412 1413 iwl_pcie_synchronize_irqs(trans); 1414 1415 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { 1416 iwl_clear_bit(trans, CSR_GP_CNTRL, 1417 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ); 1418 iwl_clear_bit(trans, CSR_GP_CNTRL, 1419 CSR_GP_CNTRL_REG_FLAG_MAC_INIT); 1420 } else { 1421 iwl_clear_bit(trans, CSR_GP_CNTRL, 1422 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 1423 iwl_clear_bit(trans, CSR_GP_CNTRL, 1424 CSR_GP_CNTRL_REG_FLAG_INIT_DONE); 1425 } 1426 1427 if (reset) { 1428 /* 1429 * reset TX queues -- some of their registers reset during S3 1430 * so if we don't reset everything here the D3 image would try 1431 * to execute some invalid memory upon resume 1432 */ 1433 iwl_trans_pcie_tx_reset(trans); 1434 } 1435 1436 iwl_pcie_set_pwr(trans, true); 1437 } 1438 1439 static int iwl_pcie_d3_handshake(struct iwl_trans *trans, bool suspend) 1440 { 1441 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1442 int ret; 1443 1444 if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) 1445 return 0; 1446 1447 trans_pcie->sx_state = IWL_SX_WAITING; 1448 1449 if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_AX210) 1450 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6, 1451 suspend ? UREG_DOORBELL_TO_ISR6_SUSPEND : 1452 UREG_DOORBELL_TO_ISR6_RESUME); 1453 else 1454 iwl_write32(trans, CSR_IPC_SLEEP_CONTROL, 1455 suspend ? CSR_IPC_SLEEP_CONTROL_SUSPEND : 1456 CSR_IPC_SLEEP_CONTROL_RESUME); 1457 1458 ret = wait_event_timeout(trans_pcie->sx_waitq, 1459 trans_pcie->sx_state != IWL_SX_WAITING, 1460 2 * HZ); 1461 if (!ret) { 1462 IWL_ERR(trans, "Timeout %s D3\n", 1463 suspend ? "entering" : "exiting"); 1464 ret = -ETIMEDOUT; 1465 } else { 1466 ret = 0; 1467 } 1468 1469 if (trans_pcie->sx_state == IWL_SX_ERROR) { 1470 IWL_ERR(trans, "FW error while %s D3\n", 1471 suspend ? "entering" : "exiting"); 1472 ret = -EIO; 1473 } 1474 1475 /* Invalidate it toward next suspend or resume */ 1476 trans_pcie->sx_state = IWL_SX_INVALID; 1477 1478 return ret; 1479 } 1480 1481 int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test, bool reset) 1482 { 1483 int ret; 1484 1485 if (!reset) 1486 /* Enable persistence mode to avoid reset */ 1487 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG, 1488 CSR_HW_IF_CONFIG_REG_PERSISTENCE); 1489 1490 ret = iwl_pcie_d3_handshake(trans, true); 1491 if (ret) 1492 return ret; 1493 1494 iwl_pcie_d3_complete_suspend(trans, test, reset); 1495 1496 return 0; 1497 } 1498 1499 int iwl_trans_pcie_d3_resume(struct iwl_trans *trans, 1500 enum iwl_d3_status *status, 1501 bool test, bool reset) 1502 { 1503 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1504 u32 val; 1505 int ret; 1506 1507 if (test) { 1508 iwl_enable_interrupts(trans); 1509 *status = IWL_D3_STATUS_ALIVE; 1510 ret = 0; 1511 goto out; 1512 } 1513 1514 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) 1515 iwl_set_bit(trans, CSR_GP_CNTRL, 1516 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ); 1517 else 1518 iwl_set_bit(trans, CSR_GP_CNTRL, 1519 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 1520 1521 ret = iwl_finish_nic_init(trans); 1522 if (ret) 1523 return ret; 1524 1525 /* 1526 * Reconfigure IVAR table in case of MSIX or reset ict table in 1527 * MSI mode since HW reset erased it. 1528 * Also enables interrupts - none will happen as 1529 * the device doesn't know we're waking it up, only when 1530 * the opmode actually tells it after this call. 1531 */ 1532 iwl_pcie_conf_msix_hw(trans_pcie); 1533 if (!trans_pcie->msix_enabled) 1534 iwl_pcie_reset_ict(trans); 1535 iwl_enable_interrupts(trans); 1536 1537 iwl_pcie_set_pwr(trans, false); 1538 1539 if (!reset) { 1540 iwl_clear_bit(trans, CSR_GP_CNTRL, 1541 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 1542 } else { 1543 iwl_trans_pcie_tx_reset(trans); 1544 1545 ret = iwl_pcie_rx_init(trans); 1546 if (ret) { 1547 IWL_ERR(trans, 1548 "Failed to resume the device (RX reset)\n"); 1549 return ret; 1550 } 1551 } 1552 1553 IWL_DEBUG_POWER(trans, "WFPM value upon resume = 0x%08X\n", 1554 iwl_read_umac_prph(trans, WFPM_GP2)); 1555 1556 val = iwl_read32(trans, CSR_RESET); 1557 if (val & CSR_RESET_REG_FLAG_NEVO_RESET) 1558 *status = IWL_D3_STATUS_RESET; 1559 else 1560 *status = IWL_D3_STATUS_ALIVE; 1561 1562 out: 1563 if (*status == IWL_D3_STATUS_ALIVE) 1564 ret = iwl_pcie_d3_handshake(trans, false); 1565 else 1566 trans->state = IWL_TRANS_NO_FW; 1567 1568 return ret; 1569 } 1570 1571 static void 1572 iwl_pcie_set_interrupt_capa(struct pci_dev *pdev, 1573 struct iwl_trans *trans, 1574 const struct iwl_mac_cfg *mac_cfg, 1575 struct iwl_trans_info *info) 1576 { 1577 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1578 int max_irqs, num_irqs, i, ret; 1579 u16 pci_cmd; 1580 u32 max_rx_queues = IWL_MAX_RX_HW_QUEUES; 1581 1582 if (!mac_cfg->mq_rx_supported) 1583 goto enable_msi; 1584 1585 if (mac_cfg->device_family <= IWL_DEVICE_FAMILY_9000) 1586 max_rx_queues = IWL_9000_MAX_RX_HW_QUEUES; 1587 1588 max_irqs = min_t(u32, num_online_cpus() + 2, max_rx_queues); 1589 for (i = 0; i < max_irqs; i++) 1590 trans_pcie->msix_entries[i].entry = i; 1591 1592 num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries, 1593 MSIX_MIN_INTERRUPT_VECTORS, 1594 max_irqs); 1595 if (num_irqs < 0) { 1596 IWL_DEBUG_INFO(trans, 1597 "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n", 1598 num_irqs); 1599 goto enable_msi; 1600 } 1601 trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0; 1602 1603 IWL_DEBUG_INFO(trans, 1604 "MSI-X enabled. %d interrupt vectors were allocated\n", 1605 num_irqs); 1606 1607 /* 1608 * In case the OS provides fewer interrupts than requested, different 1609 * causes will share the same interrupt vector as follows: 1610 * One interrupt less: non rx causes shared with FBQ. 1611 * Two interrupts less: non rx causes shared with FBQ and RSS. 1612 * More than two interrupts: we will use fewer RSS queues. 1613 */ 1614 if (num_irqs <= max_irqs - 2) { 1615 info->num_rxqs = num_irqs + 1; 1616 trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX | 1617 IWL_SHARED_IRQ_FIRST_RSS; 1618 } else if (num_irqs == max_irqs - 1) { 1619 info->num_rxqs = num_irqs; 1620 trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX; 1621 } else { 1622 info->num_rxqs = num_irqs - 1; 1623 } 1624 1625 IWL_DEBUG_INFO(trans, 1626 "MSI-X enabled with rx queues %d, vec mask 0x%x\n", 1627 info->num_rxqs, trans_pcie->shared_vec_mask); 1628 1629 WARN_ON(info->num_rxqs > IWL_MAX_RX_HW_QUEUES); 1630 1631 trans_pcie->alloc_vecs = num_irqs; 1632 trans_pcie->msix_enabled = true; 1633 return; 1634 1635 enable_msi: 1636 info->num_rxqs = 1; 1637 ret = pci_enable_msi(pdev); 1638 if (ret) { 1639 dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret); 1640 /* enable rfkill interrupt: hw bug w/a */ 1641 pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd); 1642 if (pci_cmd & PCI_COMMAND_INTX_DISABLE) { 1643 pci_cmd &= ~PCI_COMMAND_INTX_DISABLE; 1644 pci_write_config_word(pdev, PCI_COMMAND, pci_cmd); 1645 } 1646 } 1647 } 1648 1649 static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans, 1650 struct iwl_trans_info *info) 1651 { 1652 #if defined(CONFIG_SMP) 1653 int iter_rx_q, i, ret, cpu, offset; 1654 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1655 1656 i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1; 1657 iter_rx_q = info->num_rxqs - 1 + i; 1658 offset = 1 + i; 1659 for (; i < iter_rx_q ; i++) { 1660 /* 1661 * Get the cpu prior to the place to search 1662 * (i.e. return will be > i - 1). 1663 */ 1664 cpu = cpumask_next(i - offset, cpu_online_mask); 1665 cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]); 1666 ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector, 1667 &trans_pcie->affinity_mask[i]); 1668 if (ret) 1669 IWL_ERR(trans_pcie->trans, 1670 "Failed to set affinity mask for IRQ %d\n", 1671 trans_pcie->msix_entries[i].vector); 1672 } 1673 #endif 1674 } 1675 1676 static int iwl_pcie_init_msix_handler(struct pci_dev *pdev, 1677 struct iwl_trans_pcie *trans_pcie, 1678 struct iwl_trans_info *info) 1679 { 1680 int i; 1681 1682 for (i = 0; i < trans_pcie->alloc_vecs; i++) { 1683 int ret; 1684 struct msix_entry *msix_entry; 1685 const char *qname = queue_name(&pdev->dev, trans_pcie, i); 1686 1687 if (!qname) 1688 return -ENOMEM; 1689 1690 msix_entry = &trans_pcie->msix_entries[i]; 1691 ret = devm_request_threaded_irq(&pdev->dev, 1692 msix_entry->vector, 1693 iwl_pcie_msix_isr, 1694 (i == trans_pcie->def_irq) ? 1695 iwl_pcie_irq_msix_handler : 1696 iwl_pcie_irq_rx_msix_handler, 1697 IRQF_SHARED, 1698 qname, 1699 msix_entry); 1700 if (ret) { 1701 IWL_ERR(trans_pcie->trans, 1702 "Error allocating IRQ %d\n", i); 1703 1704 return ret; 1705 } 1706 } 1707 iwl_pcie_irq_set_affinity(trans_pcie->trans, info); 1708 1709 return 0; 1710 } 1711 1712 static int iwl_trans_pcie_clear_persistence_bit(struct iwl_trans *trans) 1713 { 1714 u32 hpm, wprot; 1715 1716 switch (trans->mac_cfg->device_family) { 1717 case IWL_DEVICE_FAMILY_9000: 1718 wprot = PREG_PRPH_WPROT_9000; 1719 break; 1720 case IWL_DEVICE_FAMILY_22000: 1721 wprot = PREG_PRPH_WPROT_22000; 1722 break; 1723 default: 1724 return 0; 1725 } 1726 1727 hpm = iwl_read_umac_prph_no_grab(trans, HPM_DEBUG); 1728 if (!iwl_trans_is_hw_error_value(hpm) && (hpm & PERSISTENCE_BIT)) { 1729 u32 wprot_val = iwl_read_umac_prph_no_grab(trans, wprot); 1730 1731 if (wprot_val & PREG_WFPM_ACCESS) { 1732 IWL_ERR(trans, 1733 "Error, can not clear persistence bit\n"); 1734 return -EPERM; 1735 } 1736 iwl_write_umac_prph_no_grab(trans, HPM_DEBUG, 1737 hpm & ~PERSISTENCE_BIT); 1738 } 1739 1740 return 0; 1741 } 1742 1743 static int iwl_pcie_gen2_force_power_gating(struct iwl_trans *trans) 1744 { 1745 int ret; 1746 1747 ret = iwl_finish_nic_init(trans); 1748 if (ret < 0) 1749 return ret; 1750 1751 iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG, 1752 HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE); 1753 udelay(20); 1754 iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG, 1755 HPM_HIPM_GEN_CFG_CR_PG_EN | 1756 HPM_HIPM_GEN_CFG_CR_SLP_EN); 1757 udelay(20); 1758 iwl_clear_bits_prph(trans, HPM_HIPM_GEN_CFG, 1759 HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE); 1760 1761 return iwl_trans_pcie_sw_reset(trans, true); 1762 } 1763 1764 int _iwl_trans_pcie_start_hw(struct iwl_trans *trans) 1765 { 1766 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1767 int err; 1768 1769 lockdep_assert_held(&trans_pcie->mutex); 1770 1771 err = iwl_pcie_prepare_card_hw(trans); 1772 if (err) { 1773 IWL_ERR(trans, "Error while preparing HW: %d\n", err); 1774 return err; 1775 } 1776 1777 err = iwl_trans_pcie_clear_persistence_bit(trans); 1778 if (err) 1779 return err; 1780 1781 err = iwl_trans_pcie_sw_reset(trans, true); 1782 if (err) 1783 return err; 1784 1785 if (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_22000 && 1786 trans->mac_cfg->integrated) { 1787 err = iwl_pcie_gen2_force_power_gating(trans); 1788 if (err) 1789 return err; 1790 } 1791 1792 err = iwl_pcie_apm_init(trans); 1793 if (err) 1794 return err; 1795 1796 iwl_pcie_init_msix(trans_pcie); 1797 1798 /* From now on, the op_mode will be kept updated about RF kill state */ 1799 iwl_enable_rfkill_int(trans); 1800 1801 trans_pcie->opmode_down = false; 1802 1803 /* Set is_down to false here so that...*/ 1804 trans_pcie->is_down = false; 1805 1806 /* ...rfkill can call stop_device and set it false if needed */ 1807 iwl_pcie_check_hw_rf_kill(trans); 1808 1809 return 0; 1810 } 1811 1812 int iwl_trans_pcie_start_hw(struct iwl_trans *trans) 1813 { 1814 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1815 int ret; 1816 1817 mutex_lock(&trans_pcie->mutex); 1818 ret = _iwl_trans_pcie_start_hw(trans); 1819 mutex_unlock(&trans_pcie->mutex); 1820 1821 return ret; 1822 } 1823 1824 void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans) 1825 { 1826 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1827 1828 mutex_lock(&trans_pcie->mutex); 1829 1830 /* disable interrupts - don't enable HW RF kill interrupt */ 1831 iwl_disable_interrupts(trans); 1832 1833 iwl_pcie_apm_stop(trans, true); 1834 1835 iwl_disable_interrupts(trans); 1836 1837 iwl_pcie_disable_ict(trans); 1838 1839 mutex_unlock(&trans_pcie->mutex); 1840 1841 iwl_pcie_synchronize_irqs(trans); 1842 } 1843 1844 void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val) 1845 { 1846 writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs); 1847 } 1848 1849 void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val) 1850 { 1851 writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs); 1852 } 1853 1854 u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs) 1855 { 1856 return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs); 1857 } 1858 1859 static u32 iwl_trans_pcie_prph_msk(struct iwl_trans *trans) 1860 { 1861 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) 1862 return 0x00FFFFFF; 1863 else 1864 return 0x000FFFFF; 1865 } 1866 1867 u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg) 1868 { 1869 u32 mask = iwl_trans_pcie_prph_msk(trans); 1870 1871 iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_RADDR, 1872 ((reg & mask) | (3 << 24))); 1873 return iwl_trans_pcie_read32(trans, HBUS_TARG_PRPH_RDAT); 1874 } 1875 1876 void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val) 1877 { 1878 u32 mask = iwl_trans_pcie_prph_msk(trans); 1879 1880 iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WADDR, 1881 ((addr & mask) | (3 << 24))); 1882 iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val); 1883 } 1884 1885 void iwl_trans_pcie_op_mode_enter(struct iwl_trans *trans) 1886 { 1887 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1888 1889 /* free all first - we might be reconfigured for a different size */ 1890 iwl_pcie_free_rbs_pool(trans); 1891 1892 trans_pcie->rx_page_order = 1893 iwl_trans_get_rb_size_order(trans->conf.rx_buf_size); 1894 trans_pcie->rx_buf_bytes = 1895 iwl_trans_get_rb_size(trans->conf.rx_buf_size); 1896 } 1897 1898 void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions, 1899 struct device *dev) 1900 { 1901 u8 i; 1902 struct iwl_dram_data *desc_dram = &dram_regions->prph_scratch_mem_desc; 1903 1904 /* free DRAM payloads */ 1905 for (i = 0; i < dram_regions->n_regions; i++) { 1906 dma_free_coherent(dev, dram_regions->drams[i].size, 1907 dram_regions->drams[i].block, 1908 dram_regions->drams[i].physical); 1909 } 1910 dram_regions->n_regions = 0; 1911 1912 /* free DRAM addresses array */ 1913 if (desc_dram->block) { 1914 dma_free_coherent(dev, desc_dram->size, 1915 desc_dram->block, 1916 desc_dram->physical); 1917 } 1918 memset(desc_dram, 0, sizeof(*desc_dram)); 1919 } 1920 1921 static void iwl_pcie_free_invalid_tx_cmd(struct iwl_trans *trans) 1922 { 1923 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1924 1925 iwl_pcie_free_dma_ptr(trans, &trans_pcie->invalid_tx_cmd); 1926 } 1927 1928 static int iwl_pcie_alloc_invalid_tx_cmd(struct iwl_trans *trans) 1929 { 1930 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1931 struct iwl_cmd_header_wide bad_cmd = { 1932 .cmd = INVALID_WR_PTR_CMD, 1933 .group_id = DEBUG_GROUP, 1934 .sequence = cpu_to_le16(0xffff), 1935 .length = cpu_to_le16(0), 1936 .version = 0, 1937 }; 1938 int ret; 1939 1940 ret = iwl_pcie_alloc_dma_ptr(trans, &trans_pcie->invalid_tx_cmd, 1941 sizeof(bad_cmd)); 1942 if (ret) 1943 return ret; 1944 memcpy(trans_pcie->invalid_tx_cmd.addr, &bad_cmd, sizeof(bad_cmd)); 1945 return 0; 1946 } 1947 1948 void iwl_trans_pcie_free(struct iwl_trans *trans) 1949 { 1950 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 1951 int i; 1952 1953 iwl_pcie_synchronize_irqs(trans); 1954 1955 if (trans->mac_cfg->gen2) 1956 iwl_txq_gen2_tx_free(trans); 1957 else 1958 iwl_pcie_tx_free(trans); 1959 iwl_pcie_rx_free(trans); 1960 1961 if (trans_pcie->rba.alloc_wq) { 1962 destroy_workqueue(trans_pcie->rba.alloc_wq); 1963 trans_pcie->rba.alloc_wq = NULL; 1964 } 1965 1966 if (trans_pcie->msix_enabled) { 1967 for (i = 0; i < trans_pcie->alloc_vecs; i++) { 1968 irq_set_affinity_hint( 1969 trans_pcie->msix_entries[i].vector, 1970 NULL); 1971 } 1972 1973 trans_pcie->msix_enabled = false; 1974 } else { 1975 iwl_pcie_free_ict(trans); 1976 } 1977 1978 free_netdev(trans_pcie->napi_dev); 1979 1980 iwl_pcie_free_invalid_tx_cmd(trans); 1981 1982 iwl_pcie_free_fw_monitor(trans); 1983 1984 iwl_trans_pcie_free_pnvm_dram_regions(&trans_pcie->pnvm_data, 1985 trans->dev); 1986 iwl_trans_pcie_free_pnvm_dram_regions(&trans_pcie->reduced_tables_data, 1987 trans->dev); 1988 1989 mutex_destroy(&trans_pcie->mutex); 1990 1991 if (trans_pcie->txqs.tso_hdr_page) { 1992 for_each_possible_cpu(i) { 1993 struct iwl_tso_hdr_page *p = 1994 per_cpu_ptr(trans_pcie->txqs.tso_hdr_page, i); 1995 1996 if (p && p->page) 1997 __free_page(p->page); 1998 } 1999 2000 free_percpu(trans_pcie->txqs.tso_hdr_page); 2001 } 2002 2003 iwl_trans_free(trans); 2004 } 2005 2006 static union acpi_object * 2007 iwl_trans_pcie_call_prod_reset_dsm(struct pci_dev *pdev, u16 cmd, u16 value) 2008 { 2009 #ifdef CONFIG_ACPI 2010 struct iwl_dsm_internal_product_reset_cmd pldr_arg = { 2011 .cmd = cmd, 2012 .value = value, 2013 }; 2014 union acpi_object arg = { 2015 .buffer.type = ACPI_TYPE_BUFFER, 2016 .buffer.length = sizeof(pldr_arg), 2017 .buffer.pointer = (void *)&pldr_arg, 2018 }; 2019 static const guid_t dsm_guid = GUID_INIT(0x7266172C, 0x220B, 0x4B29, 2020 0x81, 0x4F, 0x75, 0xE4, 2021 0xDD, 0x26, 0xB5, 0xFD); 2022 2023 if (!acpi_check_dsm(ACPI_HANDLE(&pdev->dev), &dsm_guid, ACPI_DSM_REV, 2024 DSM_INTERNAL_FUNC_PRODUCT_RESET)) 2025 return ERR_PTR(-ENODEV); 2026 2027 return iwl_acpi_get_dsm_object(&pdev->dev, ACPI_DSM_REV, 2028 DSM_INTERNAL_FUNC_PRODUCT_RESET, 2029 &arg, &dsm_guid); 2030 #else 2031 return ERR_PTR(-EOPNOTSUPP); 2032 #endif 2033 } 2034 2035 void iwl_trans_pcie_check_product_reset_mode(struct pci_dev *pdev) 2036 { 2037 union acpi_object *res; 2038 2039 res = iwl_trans_pcie_call_prod_reset_dsm(pdev, 2040 DSM_INTERNAL_PLDR_CMD_GET_MODE, 2041 0); 2042 if (IS_ERR(res)) 2043 return; 2044 2045 if (res->type != ACPI_TYPE_INTEGER) 2046 IWL_ERR_DEV(&pdev->dev, 2047 "unexpected return type from product reset DSM\n"); 2048 else 2049 IWL_DEBUG_DEV_POWER(&pdev->dev, 2050 "product reset mode is 0x%llx\n", 2051 res->integer.value); 2052 2053 ACPI_FREE(res); 2054 } 2055 2056 static void iwl_trans_pcie_set_product_reset(struct pci_dev *pdev, bool enable, 2057 bool integrated) 2058 { 2059 union acpi_object *res; 2060 u16 mode = enable ? DSM_INTERNAL_PLDR_MODE_EN_PROD_RESET : 0; 2061 2062 if (!integrated) 2063 mode |= DSM_INTERNAL_PLDR_MODE_EN_WIFI_FLR | 2064 DSM_INTERNAL_PLDR_MODE_EN_BT_OFF_ON; 2065 2066 res = iwl_trans_pcie_call_prod_reset_dsm(pdev, 2067 DSM_INTERNAL_PLDR_CMD_SET_MODE, 2068 mode); 2069 if (IS_ERR(res)) { 2070 if (enable) 2071 IWL_ERR_DEV(&pdev->dev, 2072 "ACPI _DSM not available (%d), cannot do product reset\n", 2073 (int)PTR_ERR(res)); 2074 return; 2075 } 2076 2077 ACPI_FREE(res); 2078 IWL_DEBUG_DEV_POWER(&pdev->dev, "%sabled product reset via DSM\n", 2079 enable ? "En" : "Dis"); 2080 iwl_trans_pcie_check_product_reset_mode(pdev); 2081 } 2082 2083 void iwl_trans_pcie_check_product_reset_status(struct pci_dev *pdev) 2084 { 2085 union acpi_object *res; 2086 2087 res = iwl_trans_pcie_call_prod_reset_dsm(pdev, 2088 DSM_INTERNAL_PLDR_CMD_GET_STATUS, 2089 0); 2090 if (IS_ERR(res)) 2091 return; 2092 2093 if (res->type != ACPI_TYPE_INTEGER) 2094 IWL_ERR_DEV(&pdev->dev, 2095 "unexpected return type from product reset DSM\n"); 2096 else 2097 IWL_DEBUG_DEV_POWER(&pdev->dev, 2098 "product reset status is 0x%llx\n", 2099 res->integer.value); 2100 2101 ACPI_FREE(res); 2102 } 2103 2104 static void iwl_trans_pcie_call_reset(struct pci_dev *pdev) 2105 { 2106 #ifdef CONFIG_ACPI 2107 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; 2108 union acpi_object *p, *ref; 2109 acpi_status status; 2110 int ret = -EINVAL; 2111 2112 status = acpi_evaluate_object(ACPI_HANDLE(&pdev->dev), 2113 "_PRR", NULL, &buffer); 2114 if (ACPI_FAILURE(status)) { 2115 IWL_DEBUG_DEV_POWER(&pdev->dev, "No _PRR method found\n"); 2116 goto out; 2117 } 2118 p = buffer.pointer; 2119 2120 if (p->type != ACPI_TYPE_PACKAGE || p->package.count != 1) { 2121 pci_err(pdev, "Bad _PRR return type\n"); 2122 goto out; 2123 } 2124 2125 ref = &p->package.elements[0]; 2126 if (ref->type != ACPI_TYPE_LOCAL_REFERENCE) { 2127 pci_err(pdev, "_PRR wasn't a reference\n"); 2128 goto out; 2129 } 2130 2131 status = acpi_evaluate_object(ref->reference.handle, 2132 "_RST", NULL, NULL); 2133 if (ACPI_FAILURE(status)) { 2134 pci_err(pdev, 2135 "Failed to call _RST on object returned by _PRR (%d)\n", 2136 status); 2137 goto out; 2138 } 2139 ret = 0; 2140 out: 2141 kfree(buffer.pointer); 2142 if (!ret) { 2143 IWL_DEBUG_DEV_POWER(&pdev->dev, "called _RST on _PRR object\n"); 2144 return; 2145 } 2146 IWL_DEBUG_DEV_POWER(&pdev->dev, 2147 "No BIOS support, using pci_reset_function()\n"); 2148 #endif 2149 pci_reset_function(pdev); 2150 } 2151 2152 struct iwl_trans_pcie_removal { 2153 struct pci_dev *pdev; 2154 struct work_struct work; 2155 enum iwl_reset_mode mode; 2156 bool integrated; 2157 }; 2158 2159 static void iwl_trans_pcie_removal_wk(struct work_struct *wk) 2160 { 2161 struct iwl_trans_pcie_removal *removal = 2162 container_of(wk, struct iwl_trans_pcie_removal, work); 2163 struct pci_dev *pdev = removal->pdev; 2164 static char *prop[] = {"EVENT=INACCESSIBLE", NULL}; 2165 struct pci_bus *bus; 2166 2167 pci_lock_rescan_remove(); 2168 2169 bus = pdev->bus; 2170 /* in this case, something else already removed the device */ 2171 if (!bus) 2172 goto out; 2173 2174 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop); 2175 2176 if (removal->mode == IWL_RESET_MODE_PROD_RESET) { 2177 struct pci_dev *bt = NULL; 2178 2179 if (!removal->integrated) { 2180 /* discrete devices have WiFi/BT at function 0/1 */ 2181 int slot = PCI_SLOT(pdev->devfn); 2182 int func = PCI_FUNC(pdev->devfn); 2183 2184 if (func == 0) 2185 bt = pci_get_slot(bus, PCI_DEVFN(slot, 1)); 2186 else 2187 pci_info(pdev, "Unexpected function %d\n", 2188 func); 2189 } else { 2190 /* on integrated we have to look up by ID (same bus) */ 2191 static const struct pci_device_id bt_device_ids[] = { 2192 #define BT_DEV(_id) { PCI_DEVICE(PCI_VENDOR_ID_INTEL, _id) } 2193 BT_DEV(0xA876), /* LNL */ 2194 BT_DEV(0xE476), /* PTL-P */ 2195 BT_DEV(0xE376), /* PTL-H */ 2196 BT_DEV(0xD346), /* NVL-H */ 2197 BT_DEV(0x6E74), /* NVL-S */ 2198 BT_DEV(0x4D76), /* WCL */ 2199 BT_DEV(0xD246), /* RZL-H */ 2200 BT_DEV(0x6C46), /* RZL-M */ 2201 {} 2202 }; 2203 struct pci_dev *tmp = NULL; 2204 2205 for_each_pci_dev(tmp) { 2206 if (tmp->bus != bus) 2207 continue; 2208 2209 if (pci_match_id(bt_device_ids, tmp)) { 2210 bt = tmp; 2211 break; 2212 } 2213 } 2214 } 2215 2216 if (bt) { 2217 pci_info(bt, "Removal by WiFi due to product reset\n"); 2218 pci_stop_and_remove_bus_device(bt); 2219 pci_dev_put(bt); 2220 } 2221 } 2222 2223 iwl_trans_pcie_set_product_reset(pdev, 2224 removal->mode == 2225 IWL_RESET_MODE_PROD_RESET, 2226 removal->integrated); 2227 if (removal->mode >= IWL_RESET_MODE_FUNC_RESET) 2228 iwl_trans_pcie_call_reset(pdev); 2229 2230 pci_stop_and_remove_bus_device(pdev); 2231 pci_dev_put(pdev); 2232 2233 if (removal->mode >= IWL_RESET_MODE_RESCAN) { 2234 if (bus->parent) 2235 bus = bus->parent; 2236 pci_rescan_bus(bus); 2237 } 2238 2239 out: 2240 pci_unlock_rescan_remove(); 2241 2242 kfree(removal); 2243 module_put(THIS_MODULE); 2244 } 2245 2246 void iwl_trans_pcie_reset(struct iwl_trans *trans, enum iwl_reset_mode mode) 2247 { 2248 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2249 struct iwl_trans_pcie_removal *removal; 2250 char _msg = 0, *msg = &_msg; 2251 2252 if (WARN_ON(mode < IWL_RESET_MODE_REMOVE_ONLY || 2253 mode == IWL_RESET_MODE_BACKOFF)) 2254 return; 2255 2256 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 2257 return; 2258 2259 if (trans_pcie->me_present && mode == IWL_RESET_MODE_PROD_RESET) { 2260 mode = IWL_RESET_MODE_FUNC_RESET; 2261 if (trans_pcie->me_present < 0) 2262 msg = " instead of product reset as ME may be present"; 2263 else 2264 msg = " instead of product reset as ME is present"; 2265 } 2266 2267 IWL_INFO(trans, "scheduling reset (mode=%d%s)\n", mode, msg); 2268 2269 iwl_pcie_dump_csr(trans); 2270 2271 /* 2272 * get a module reference to avoid doing this 2273 * while unloading anyway and to avoid 2274 * scheduling a work with code that's being 2275 * removed. 2276 */ 2277 if (!try_module_get(THIS_MODULE)) { 2278 IWL_ERR(trans, 2279 "Module is being unloaded - abort\n"); 2280 return; 2281 } 2282 2283 removal = kzalloc(sizeof(*removal), GFP_ATOMIC); 2284 if (!removal) { 2285 module_put(THIS_MODULE); 2286 return; 2287 } 2288 /* 2289 * we don't need to clear this flag, because 2290 * the trans will be freed and reallocated. 2291 */ 2292 set_bit(STATUS_TRANS_DEAD, &trans->status); 2293 2294 removal->pdev = to_pci_dev(trans->dev); 2295 removal->mode = mode; 2296 removal->integrated = trans->mac_cfg->integrated; 2297 INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk); 2298 pci_dev_get(removal->pdev); 2299 schedule_work(&removal->work); 2300 } 2301 EXPORT_SYMBOL(iwl_trans_pcie_reset); 2302 2303 /* 2304 * This version doesn't disable BHs but rather assumes they're 2305 * already disabled. 2306 */ 2307 bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent) 2308 { 2309 int ret; 2310 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2311 u32 write = CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ; 2312 u32 mask = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | 2313 CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP; 2314 u32 poll = CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN; 2315 2316 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 2317 return false; 2318 2319 spin_lock(&trans_pcie->reg_lock); 2320 2321 if (trans_pcie->cmd_hold_nic_awake) 2322 goto out; 2323 2324 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) { 2325 write = CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ; 2326 mask = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS; 2327 poll = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS; 2328 } 2329 2330 /* this bit wakes up the NIC */ 2331 iwl_trans_set_bit(trans, CSR_GP_CNTRL, write); 2332 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000) 2333 udelay(2); 2334 2335 /* 2336 * These bits say the device is running, and should keep running for 2337 * at least a short while (at least as long as MAC_ACCESS_REQ stays 1), 2338 * but they do not indicate that embedded SRAM is restored yet; 2339 * HW with volatile SRAM must save/restore contents to/from 2340 * host DRAM when sleeping/waking for power-saving. 2341 * Each direction takes approximately 1/4 millisecond; with this 2342 * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a 2343 * series of register accesses are expected (e.g. reading Event Log), 2344 * to keep device from sleeping. 2345 * 2346 * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that 2347 * SRAM is okay/restored. We don't check that here because this call 2348 * is just for hardware register access; but GP1 MAC_SLEEP 2349 * check is a good idea before accessing the SRAM of HW with 2350 * volatile SRAM (e.g. reading Event Log). 2351 * 2352 * 5000 series and later (including 1000 series) have non-volatile SRAM, 2353 * and do not save/restore SRAM when power cycling. 2354 */ 2355 ret = iwl_poll_bits_mask(trans, CSR_GP_CNTRL, poll, mask, 15000); 2356 if (unlikely(ret < 0)) { 2357 u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL); 2358 2359 if (silent) { 2360 spin_unlock(&trans_pcie->reg_lock); 2361 return false; 2362 } 2363 2364 WARN_ONCE(1, 2365 "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n", 2366 cntrl); 2367 2368 iwl_trans_pcie_dump_regs(trans, trans_pcie->pci_dev); 2369 2370 if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U) 2371 iwl_trans_pcie_reset(trans, 2372 IWL_RESET_MODE_REMOVE_ONLY); 2373 else 2374 iwl_write32(trans, CSR_RESET, 2375 CSR_RESET_REG_FLAG_FORCE_NMI); 2376 2377 spin_unlock(&trans_pcie->reg_lock); 2378 return false; 2379 } 2380 2381 out: 2382 /* 2383 * Fool sparse by faking we release the lock - sparse will 2384 * track nic_access anyway. 2385 */ 2386 __release(&trans_pcie->reg_lock); 2387 return true; 2388 } 2389 2390 bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans) 2391 { 2392 bool ret; 2393 2394 local_bh_disable(); 2395 ret = __iwl_trans_pcie_grab_nic_access(trans, false); 2396 if (ret) { 2397 /* keep BHs disabled until iwl_trans_pcie_release_nic_access */ 2398 return ret; 2399 } 2400 local_bh_enable(); 2401 return false; 2402 } 2403 2404 void __releases(nic_access_nobh) 2405 iwl_trans_pcie_release_nic_access(struct iwl_trans *trans) 2406 { 2407 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2408 2409 lockdep_assert_held(&trans_pcie->reg_lock); 2410 2411 /* 2412 * Fool sparse by faking we acquiring the lock - sparse will 2413 * track nic_access anyway. 2414 */ 2415 __acquire(&trans_pcie->reg_lock); 2416 2417 if (trans_pcie->cmd_hold_nic_awake) 2418 goto out; 2419 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) 2420 iwl_trans_clear_bit(trans, CSR_GP_CNTRL, 2421 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ); 2422 else 2423 iwl_trans_clear_bit(trans, CSR_GP_CNTRL, 2424 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); 2425 /* 2426 * Above we read the CSR_GP_CNTRL register, which will flush 2427 * any previous writes, but we need the write that clears the 2428 * MAC_ACCESS_REQ bit to be performed before any other writes 2429 * scheduled on different CPUs (after we drop reg_lock). 2430 */ 2431 out: 2432 __release(nic_access_nobh); 2433 spin_unlock_bh(&trans_pcie->reg_lock); 2434 } 2435 2436 int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr, 2437 void *buf, int dwords) 2438 { 2439 #define IWL_MAX_HW_ERRS 5 2440 unsigned int num_consec_hw_errors = 0; 2441 int offs = 0; 2442 u32 *vals = buf; 2443 2444 while (offs < dwords) { 2445 /* limit the time we spin here under lock to 1/2s */ 2446 unsigned long end = jiffies + HZ / 2; 2447 bool resched = false; 2448 2449 if (iwl_trans_grab_nic_access(trans)) { 2450 iwl_write32(trans, HBUS_TARG_MEM_RADDR, 2451 addr + 4 * offs); 2452 2453 while (offs < dwords) { 2454 vals[offs] = iwl_read32(trans, 2455 HBUS_TARG_MEM_RDAT); 2456 2457 if (iwl_trans_is_hw_error_value(vals[offs])) 2458 num_consec_hw_errors++; 2459 else 2460 num_consec_hw_errors = 0; 2461 2462 if (num_consec_hw_errors >= IWL_MAX_HW_ERRS) { 2463 iwl_trans_release_nic_access(trans); 2464 return -EIO; 2465 } 2466 2467 offs++; 2468 2469 if (time_after(jiffies, end)) { 2470 resched = true; 2471 break; 2472 } 2473 } 2474 iwl_trans_release_nic_access(trans); 2475 2476 if (resched) 2477 cond_resched(); 2478 } else { 2479 return -EBUSY; 2480 } 2481 } 2482 2483 return 0; 2484 } 2485 2486 int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs, 2487 u32 *val) 2488 { 2489 return pci_read_config_dword(IWL_TRANS_GET_PCIE_TRANS(trans)->pci_dev, 2490 ofs, val); 2491 } 2492 2493 #define IWL_FLUSH_WAIT_MS 2000 2494 2495 int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue, 2496 struct iwl_trans_rxq_dma_data *data) 2497 { 2498 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2499 2500 if (queue >= trans->info.num_rxqs || !trans_pcie->rxq) 2501 return -EINVAL; 2502 2503 data->fr_bd_cb = trans_pcie->rxq[queue].bd_dma; 2504 data->urbd_stts_wrptr = trans_pcie->rxq[queue].rb_stts_dma; 2505 data->ur_bd_cb = trans_pcie->rxq[queue].used_bd_dma; 2506 data->fr_bd_wid = 0; 2507 2508 return 0; 2509 } 2510 2511 int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx) 2512 { 2513 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2514 struct iwl_txq *txq; 2515 unsigned long now = jiffies; 2516 bool overflow_tx; 2517 u8 wr_ptr; 2518 2519 /* Make sure the NIC is still alive in the bus */ 2520 if (test_bit(STATUS_TRANS_DEAD, &trans->status)) 2521 return -ENODEV; 2522 2523 if (!test_bit(txq_idx, trans_pcie->txqs.queue_used)) 2524 return -EINVAL; 2525 2526 IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", txq_idx); 2527 txq = trans_pcie->txqs.txq[txq_idx]; 2528 2529 spin_lock_bh(&txq->lock); 2530 overflow_tx = txq->overflow_tx || 2531 !skb_queue_empty(&txq->overflow_q); 2532 spin_unlock_bh(&txq->lock); 2533 2534 wr_ptr = READ_ONCE(txq->write_ptr); 2535 2536 while ((txq->read_ptr != READ_ONCE(txq->write_ptr) || 2537 overflow_tx) && 2538 !time_after(jiffies, 2539 now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) { 2540 u8 write_ptr = READ_ONCE(txq->write_ptr); 2541 2542 /* 2543 * If write pointer moved during the wait, warn only 2544 * if the TX came from op mode. In case TX came from 2545 * trans layer (overflow TX) don't warn. 2546 */ 2547 if (WARN_ONCE(wr_ptr != write_ptr && !overflow_tx, 2548 "WR pointer moved while flushing %d -> %d\n", 2549 wr_ptr, write_ptr)) 2550 return -ETIMEDOUT; 2551 wr_ptr = write_ptr; 2552 2553 usleep_range(1000, 2000); 2554 2555 spin_lock_bh(&txq->lock); 2556 overflow_tx = txq->overflow_tx || 2557 !skb_queue_empty(&txq->overflow_q); 2558 spin_unlock_bh(&txq->lock); 2559 } 2560 2561 if (txq->read_ptr != txq->write_ptr) { 2562 IWL_ERR(trans, 2563 "fail to flush all tx fifo queues Q %d\n", txq_idx); 2564 iwl_txq_log_scd_error(trans, txq); 2565 return -ETIMEDOUT; 2566 } 2567 2568 IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", txq_idx); 2569 2570 return 0; 2571 } 2572 2573 int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm) 2574 { 2575 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2576 int cnt; 2577 int ret = 0; 2578 2579 /* waiting for all the tx frames complete might take a while */ 2580 for (cnt = 0; 2581 cnt < trans->mac_cfg->base->num_of_queues; 2582 cnt++) { 2583 2584 if (cnt == trans->conf.cmd_queue) 2585 continue; 2586 if (!test_bit(cnt, trans_pcie->txqs.queue_used)) 2587 continue; 2588 if (!(BIT(cnt) & txq_bm)) 2589 continue; 2590 2591 ret = iwl_trans_pcie_wait_txq_empty(trans, cnt); 2592 if (ret) 2593 break; 2594 } 2595 2596 return ret; 2597 } 2598 2599 void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg, 2600 u32 mask, u32 value) 2601 { 2602 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2603 2604 spin_lock_bh(&trans_pcie->reg_lock); 2605 _iwl_trans_set_bits_mask(trans, reg, mask, value); 2606 spin_unlock_bh(&trans_pcie->reg_lock); 2607 } 2608 2609 static const char *get_csr_string(int cmd) 2610 { 2611 #define IWL_CMD(x) case x: return #x 2612 switch (cmd) { 2613 IWL_CMD(CSR_HW_IF_CONFIG_REG); 2614 IWL_CMD(CSR_INT_COALESCING); 2615 IWL_CMD(CSR_INT); 2616 IWL_CMD(CSR_INT_MASK); 2617 IWL_CMD(CSR_FH_INT_STATUS); 2618 IWL_CMD(CSR_GPIO_IN); 2619 IWL_CMD(CSR_RESET); 2620 IWL_CMD(CSR_GP_CNTRL); 2621 IWL_CMD(CSR_HW_REV); 2622 IWL_CMD(CSR_EEPROM_REG); 2623 IWL_CMD(CSR_EEPROM_GP); 2624 IWL_CMD(CSR_OTP_GP_REG); 2625 IWL_CMD(CSR_GIO_REG); 2626 IWL_CMD(CSR_GP_UCODE_REG); 2627 IWL_CMD(CSR_GP_DRIVER_REG); 2628 IWL_CMD(CSR_UCODE_DRV_GP1); 2629 IWL_CMD(CSR_UCODE_DRV_GP2); 2630 IWL_CMD(CSR_LED_REG); 2631 IWL_CMD(CSR_DRAM_INT_TBL_REG); 2632 IWL_CMD(CSR_GIO_CHICKEN_BITS); 2633 IWL_CMD(CSR_ANA_PLL_CFG); 2634 IWL_CMD(CSR_HW_REV_WA_REG); 2635 IWL_CMD(CSR_MONITOR_STATUS_REG); 2636 IWL_CMD(CSR_DBG_HPET_MEM_REG); 2637 default: 2638 return "UNKNOWN"; 2639 } 2640 #undef IWL_CMD 2641 } 2642 2643 void iwl_pcie_dump_csr(struct iwl_trans *trans) 2644 { 2645 int i; 2646 static const u32 csr_tbl[] = { 2647 CSR_HW_IF_CONFIG_REG, 2648 CSR_INT_COALESCING, 2649 CSR_INT, 2650 CSR_INT_MASK, 2651 CSR_FH_INT_STATUS, 2652 CSR_GPIO_IN, 2653 CSR_RESET, 2654 CSR_GP_CNTRL, 2655 CSR_HW_REV, 2656 CSR_EEPROM_REG, 2657 CSR_EEPROM_GP, 2658 CSR_OTP_GP_REG, 2659 CSR_GIO_REG, 2660 CSR_GP_UCODE_REG, 2661 CSR_GP_DRIVER_REG, 2662 CSR_UCODE_DRV_GP1, 2663 CSR_UCODE_DRV_GP2, 2664 CSR_LED_REG, 2665 CSR_DRAM_INT_TBL_REG, 2666 CSR_GIO_CHICKEN_BITS, 2667 CSR_ANA_PLL_CFG, 2668 CSR_MONITOR_STATUS_REG, 2669 CSR_HW_REV_WA_REG, 2670 CSR_DBG_HPET_MEM_REG 2671 }; 2672 IWL_ERR(trans, "CSR values:\n"); 2673 IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is " 2674 "CSR_INT_PERIODIC_REG)\n"); 2675 for (i = 0; i < ARRAY_SIZE(csr_tbl); i++) { 2676 IWL_ERR(trans, " %25s: 0X%08x\n", 2677 get_csr_string(csr_tbl[i]), 2678 iwl_read32(trans, csr_tbl[i])); 2679 } 2680 } 2681 2682 #ifdef CONFIG_IWLWIFI_DEBUGFS 2683 /* create and remove of files */ 2684 #define DEBUGFS_ADD_FILE(name, parent, mode) do { \ 2685 debugfs_create_file(#name, mode, parent, trans, \ 2686 &iwl_dbgfs_##name##_ops); \ 2687 } while (0) 2688 2689 /* file operation */ 2690 #define DEBUGFS_READ_FILE_OPS(name) \ 2691 static const struct file_operations iwl_dbgfs_##name##_ops = { \ 2692 .read = iwl_dbgfs_##name##_read, \ 2693 .open = simple_open, \ 2694 .llseek = generic_file_llseek, \ 2695 }; 2696 2697 #define DEBUGFS_WRITE_FILE_OPS(name) \ 2698 static const struct file_operations iwl_dbgfs_##name##_ops = { \ 2699 .write = iwl_dbgfs_##name##_write, \ 2700 .open = simple_open, \ 2701 .llseek = generic_file_llseek, \ 2702 }; 2703 2704 #define DEBUGFS_READ_WRITE_FILE_OPS(name) \ 2705 static const struct file_operations iwl_dbgfs_##name##_ops = { \ 2706 .write = iwl_dbgfs_##name##_write, \ 2707 .read = iwl_dbgfs_##name##_read, \ 2708 .open = simple_open, \ 2709 .llseek = generic_file_llseek, \ 2710 }; 2711 2712 struct iwl_dbgfs_tx_queue_priv { 2713 struct iwl_trans *trans; 2714 }; 2715 2716 struct iwl_dbgfs_tx_queue_state { 2717 loff_t pos; 2718 }; 2719 2720 static void *iwl_dbgfs_tx_queue_seq_start(struct seq_file *seq, loff_t *pos) 2721 { 2722 struct iwl_dbgfs_tx_queue_priv *priv = seq->private; 2723 struct iwl_dbgfs_tx_queue_state *state; 2724 2725 if (*pos >= priv->trans->mac_cfg->base->num_of_queues) 2726 return NULL; 2727 2728 state = kmalloc(sizeof(*state), GFP_KERNEL); 2729 if (!state) 2730 return NULL; 2731 state->pos = *pos; 2732 return state; 2733 } 2734 2735 static void *iwl_dbgfs_tx_queue_seq_next(struct seq_file *seq, 2736 void *v, loff_t *pos) 2737 { 2738 struct iwl_dbgfs_tx_queue_priv *priv = seq->private; 2739 struct iwl_dbgfs_tx_queue_state *state = v; 2740 2741 *pos = ++state->pos; 2742 2743 if (*pos >= priv->trans->mac_cfg->base->num_of_queues) 2744 return NULL; 2745 2746 return state; 2747 } 2748 2749 static void iwl_dbgfs_tx_queue_seq_stop(struct seq_file *seq, void *v) 2750 { 2751 kfree(v); 2752 } 2753 2754 static int iwl_dbgfs_tx_queue_seq_show(struct seq_file *seq, void *v) 2755 { 2756 struct iwl_dbgfs_tx_queue_priv *priv = seq->private; 2757 struct iwl_dbgfs_tx_queue_state *state = v; 2758 struct iwl_trans *trans = priv->trans; 2759 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2760 struct iwl_txq *txq = trans_pcie->txqs.txq[state->pos]; 2761 2762 seq_printf(seq, "hwq %.3u: used=%d stopped=%d ", 2763 (unsigned int)state->pos, 2764 !!test_bit(state->pos, trans_pcie->txqs.queue_used), 2765 !!test_bit(state->pos, trans_pcie->txqs.queue_stopped)); 2766 if (txq) 2767 seq_printf(seq, 2768 "read=%u write=%u need_update=%d frozen=%d n_window=%d ampdu=%d", 2769 txq->read_ptr, txq->write_ptr, 2770 txq->need_update, txq->frozen, 2771 txq->n_window, txq->ampdu); 2772 else 2773 seq_puts(seq, "(unallocated)"); 2774 2775 if (state->pos == trans->conf.cmd_queue) 2776 seq_puts(seq, " (HCMD)"); 2777 seq_puts(seq, "\n"); 2778 2779 return 0; 2780 } 2781 2782 static const struct seq_operations iwl_dbgfs_tx_queue_seq_ops = { 2783 .start = iwl_dbgfs_tx_queue_seq_start, 2784 .next = iwl_dbgfs_tx_queue_seq_next, 2785 .stop = iwl_dbgfs_tx_queue_seq_stop, 2786 .show = iwl_dbgfs_tx_queue_seq_show, 2787 }; 2788 2789 static int iwl_dbgfs_tx_queue_open(struct inode *inode, struct file *filp) 2790 { 2791 struct iwl_dbgfs_tx_queue_priv *priv; 2792 2793 priv = __seq_open_private(filp, &iwl_dbgfs_tx_queue_seq_ops, 2794 sizeof(*priv)); 2795 2796 if (!priv) 2797 return -ENOMEM; 2798 2799 priv->trans = inode->i_private; 2800 return 0; 2801 } 2802 2803 static ssize_t iwl_dbgfs_rx_queue_read(struct file *file, 2804 char __user *user_buf, 2805 size_t count, loff_t *ppos) 2806 { 2807 struct iwl_trans *trans = file->private_data; 2808 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2809 char *buf; 2810 int pos = 0, i, ret; 2811 size_t bufsz; 2812 2813 bufsz = sizeof(char) * 121 * trans->info.num_rxqs; 2814 2815 if (!trans_pcie->rxq) 2816 return -EAGAIN; 2817 2818 buf = kzalloc(bufsz, GFP_KERNEL); 2819 if (!buf) 2820 return -ENOMEM; 2821 2822 for (i = 0; i < trans->info.num_rxqs && pos < bufsz; i++) { 2823 struct iwl_rxq *rxq = &trans_pcie->rxq[i]; 2824 2825 spin_lock_bh(&rxq->lock); 2826 2827 pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n", 2828 i); 2829 pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n", 2830 rxq->read); 2831 pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n", 2832 rxq->write); 2833 pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n", 2834 rxq->write_actual); 2835 pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n", 2836 rxq->need_update); 2837 pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n", 2838 rxq->free_count); 2839 if (rxq->rb_stts) { 2840 u32 r = iwl_get_closed_rb_stts(trans, rxq); 2841 pos += scnprintf(buf + pos, bufsz - pos, 2842 "\tclosed_rb_num: %u\n", r); 2843 } else { 2844 pos += scnprintf(buf + pos, bufsz - pos, 2845 "\tclosed_rb_num: Not Allocated\n"); 2846 } 2847 spin_unlock_bh(&rxq->lock); 2848 } 2849 ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos); 2850 kfree(buf); 2851 2852 return ret; 2853 } 2854 2855 static ssize_t iwl_dbgfs_interrupt_read(struct file *file, 2856 char __user *user_buf, 2857 size_t count, loff_t *ppos) 2858 { 2859 struct iwl_trans *trans = file->private_data; 2860 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2861 struct isr_statistics *isr_stats = &trans_pcie->isr_stats; 2862 2863 int pos = 0; 2864 char *buf; 2865 int bufsz = 24 * 64; /* 24 items * 64 char per item */ 2866 ssize_t ret; 2867 2868 buf = kzalloc(bufsz, GFP_KERNEL); 2869 if (!buf) 2870 return -ENOMEM; 2871 2872 pos += scnprintf(buf + pos, bufsz - pos, 2873 "Interrupt Statistics Report:\n"); 2874 2875 pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n", 2876 isr_stats->hw); 2877 pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n", 2878 isr_stats->sw); 2879 if (isr_stats->sw || isr_stats->hw) { 2880 pos += scnprintf(buf + pos, bufsz - pos, 2881 "\tLast Restarting Code: 0x%X\n", 2882 isr_stats->err_code); 2883 } 2884 #ifdef CONFIG_IWLWIFI_DEBUG 2885 pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n", 2886 isr_stats->sch); 2887 pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n", 2888 isr_stats->alive); 2889 #endif 2890 pos += scnprintf(buf + pos, bufsz - pos, 2891 "HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill); 2892 2893 pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n", 2894 isr_stats->ctkill); 2895 2896 pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n", 2897 isr_stats->wakeup); 2898 2899 pos += scnprintf(buf + pos, bufsz - pos, 2900 "Rx command responses:\t\t %u\n", isr_stats->rx); 2901 2902 pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n", 2903 isr_stats->tx); 2904 2905 pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n", 2906 isr_stats->unhandled); 2907 2908 ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos); 2909 kfree(buf); 2910 return ret; 2911 } 2912 2913 static ssize_t iwl_dbgfs_interrupt_write(struct file *file, 2914 const char __user *user_buf, 2915 size_t count, loff_t *ppos) 2916 { 2917 struct iwl_trans *trans = file->private_data; 2918 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2919 struct isr_statistics *isr_stats = &trans_pcie->isr_stats; 2920 u32 reset_flag; 2921 int ret; 2922 2923 ret = kstrtou32_from_user(user_buf, count, 16, &reset_flag); 2924 if (ret) 2925 return ret; 2926 if (reset_flag == 0) 2927 memset(isr_stats, 0, sizeof(*isr_stats)); 2928 2929 return count; 2930 } 2931 2932 static ssize_t iwl_dbgfs_csr_write(struct file *file, 2933 const char __user *user_buf, 2934 size_t count, loff_t *ppos) 2935 { 2936 struct iwl_trans *trans = file->private_data; 2937 2938 iwl_pcie_dump_csr(trans); 2939 2940 return count; 2941 } 2942 2943 static ssize_t iwl_dbgfs_fh_reg_read(struct file *file, 2944 char __user *user_buf, 2945 size_t count, loff_t *ppos) 2946 { 2947 struct iwl_trans *trans = file->private_data; 2948 char *buf = NULL; 2949 ssize_t ret; 2950 2951 ret = iwl_dump_fh(trans, &buf); 2952 if (ret < 0) 2953 return ret; 2954 if (!buf) 2955 return -EINVAL; 2956 ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret); 2957 kfree(buf); 2958 return ret; 2959 } 2960 2961 static ssize_t iwl_dbgfs_rfkill_read(struct file *file, 2962 char __user *user_buf, 2963 size_t count, loff_t *ppos) 2964 { 2965 struct iwl_trans *trans = file->private_data; 2966 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2967 char buf[100]; 2968 int pos; 2969 2970 pos = scnprintf(buf, sizeof(buf), "debug: %d\nhw: %d\n", 2971 trans_pcie->debug_rfkill, 2972 !(iwl_read32(trans, CSR_GP_CNTRL) & 2973 CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW)); 2974 2975 return simple_read_from_buffer(user_buf, count, ppos, buf, pos); 2976 } 2977 2978 static ssize_t iwl_dbgfs_rfkill_write(struct file *file, 2979 const char __user *user_buf, 2980 size_t count, loff_t *ppos) 2981 { 2982 struct iwl_trans *trans = file->private_data; 2983 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 2984 bool new_value; 2985 int ret; 2986 2987 ret = kstrtobool_from_user(user_buf, count, &new_value); 2988 if (ret) 2989 return ret; 2990 if (new_value == trans_pcie->debug_rfkill) 2991 return count; 2992 IWL_WARN(trans, "changing debug rfkill %d->%d\n", 2993 trans_pcie->debug_rfkill, new_value); 2994 trans_pcie->debug_rfkill = new_value; 2995 iwl_pcie_handle_rfkill_irq(trans, false); 2996 2997 return count; 2998 } 2999 3000 static int iwl_dbgfs_monitor_data_open(struct inode *inode, 3001 struct file *file) 3002 { 3003 struct iwl_trans *trans = inode->i_private; 3004 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3005 3006 if (!trans->dbg.dest_tlv || 3007 trans->dbg.dest_tlv->monitor_mode != EXTERNAL_MODE) { 3008 IWL_ERR(trans, "Debug destination is not set to DRAM\n"); 3009 return -ENOENT; 3010 } 3011 3012 if (trans_pcie->fw_mon_data.state != IWL_FW_MON_DBGFS_STATE_CLOSED) 3013 return -EBUSY; 3014 3015 trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_OPEN; 3016 return simple_open(inode, file); 3017 } 3018 3019 static int iwl_dbgfs_monitor_data_release(struct inode *inode, 3020 struct file *file) 3021 { 3022 struct iwl_trans_pcie *trans_pcie = 3023 IWL_TRANS_GET_PCIE_TRANS(inode->i_private); 3024 3025 if (trans_pcie->fw_mon_data.state == IWL_FW_MON_DBGFS_STATE_OPEN) 3026 trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED; 3027 return 0; 3028 } 3029 3030 static bool iwl_write_to_user_buf(char __user *user_buf, ssize_t count, 3031 void *buf, ssize_t *size, 3032 ssize_t *bytes_copied) 3033 { 3034 ssize_t buf_size_left = count - *bytes_copied; 3035 3036 buf_size_left = buf_size_left - (buf_size_left % sizeof(u32)); 3037 if (*size > buf_size_left) 3038 *size = buf_size_left; 3039 3040 *size -= copy_to_user(user_buf, buf, *size); 3041 *bytes_copied += *size; 3042 3043 if (buf_size_left == *size) 3044 return true; 3045 return false; 3046 } 3047 3048 static ssize_t iwl_dbgfs_monitor_data_read(struct file *file, 3049 char __user *user_buf, 3050 size_t count, loff_t *ppos) 3051 { 3052 struct iwl_trans *trans = file->private_data; 3053 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3054 u8 *cpu_addr = (void *)trans->dbg.fw_mon.block, *curr_buf; 3055 struct cont_rec *data = &trans_pcie->fw_mon_data; 3056 u32 write_ptr_addr, wrap_cnt_addr, write_ptr, wrap_cnt; 3057 ssize_t size, bytes_copied = 0; 3058 bool b_full; 3059 3060 if (trans->dbg.dest_tlv) { 3061 write_ptr_addr = 3062 le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg); 3063 wrap_cnt_addr = le32_to_cpu(trans->dbg.dest_tlv->wrap_count); 3064 } else { 3065 write_ptr_addr = MON_BUFF_WRPTR; 3066 wrap_cnt_addr = MON_BUFF_CYCLE_CNT; 3067 } 3068 3069 if (unlikely(!trans->dbg.rec_on)) 3070 return 0; 3071 3072 mutex_lock(&data->mutex); 3073 if (data->state == 3074 IWL_FW_MON_DBGFS_STATE_DISABLED) { 3075 mutex_unlock(&data->mutex); 3076 return 0; 3077 } 3078 3079 /* write_ptr position in bytes rather then DW */ 3080 write_ptr = iwl_read_prph(trans, write_ptr_addr) * sizeof(u32); 3081 wrap_cnt = iwl_read_prph(trans, wrap_cnt_addr); 3082 3083 if (data->prev_wrap_cnt == wrap_cnt) { 3084 size = write_ptr - data->prev_wr_ptr; 3085 curr_buf = cpu_addr + data->prev_wr_ptr; 3086 b_full = iwl_write_to_user_buf(user_buf, count, 3087 curr_buf, &size, 3088 &bytes_copied); 3089 data->prev_wr_ptr += size; 3090 3091 } else if (data->prev_wrap_cnt == wrap_cnt - 1 && 3092 write_ptr < data->prev_wr_ptr) { 3093 size = trans->dbg.fw_mon.size - data->prev_wr_ptr; 3094 curr_buf = cpu_addr + data->prev_wr_ptr; 3095 b_full = iwl_write_to_user_buf(user_buf, count, 3096 curr_buf, &size, 3097 &bytes_copied); 3098 data->prev_wr_ptr += size; 3099 3100 if (!b_full) { 3101 size = write_ptr; 3102 b_full = iwl_write_to_user_buf(user_buf, count, 3103 cpu_addr, &size, 3104 &bytes_copied); 3105 data->prev_wr_ptr = size; 3106 data->prev_wrap_cnt++; 3107 } 3108 } else { 3109 if (data->prev_wrap_cnt == wrap_cnt - 1 && 3110 write_ptr > data->prev_wr_ptr) 3111 IWL_WARN(trans, 3112 "write pointer passed previous write pointer, start copying from the beginning\n"); 3113 else if (!unlikely(data->prev_wrap_cnt == 0 && 3114 data->prev_wr_ptr == 0)) 3115 IWL_WARN(trans, 3116 "monitor data is out of sync, start copying from the beginning\n"); 3117 3118 size = write_ptr; 3119 b_full = iwl_write_to_user_buf(user_buf, count, 3120 cpu_addr, &size, 3121 &bytes_copied); 3122 data->prev_wr_ptr = size; 3123 data->prev_wrap_cnt = wrap_cnt; 3124 } 3125 3126 mutex_unlock(&data->mutex); 3127 3128 return bytes_copied; 3129 } 3130 3131 static ssize_t iwl_dbgfs_rf_read(struct file *file, 3132 char __user *user_buf, 3133 size_t count, loff_t *ppos) 3134 { 3135 struct iwl_trans *trans = file->private_data; 3136 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3137 3138 if (!trans_pcie->rf_name[0]) 3139 return -ENODEV; 3140 3141 return simple_read_from_buffer(user_buf, count, ppos, 3142 trans_pcie->rf_name, 3143 strlen(trans_pcie->rf_name)); 3144 } 3145 3146 static ssize_t iwl_dbgfs_reset_write(struct file *file, 3147 const char __user *user_buf, 3148 size_t count, loff_t *ppos) 3149 { 3150 struct iwl_trans *trans = file->private_data; 3151 static const char * const modes[] = { 3152 [IWL_RESET_MODE_SW_RESET] = "sw", 3153 [IWL_RESET_MODE_REPROBE] = "reprobe", 3154 [IWL_RESET_MODE_TOP_RESET] = "top", 3155 [IWL_RESET_MODE_REMOVE_ONLY] = "remove", 3156 [IWL_RESET_MODE_RESCAN] = "rescan", 3157 [IWL_RESET_MODE_FUNC_RESET] = "function", 3158 [IWL_RESET_MODE_PROD_RESET] = "product", 3159 }; 3160 char buf[10] = {}; 3161 int mode; 3162 3163 if (count > sizeof(buf) - 1) 3164 return -EINVAL; 3165 3166 if (copy_from_user(buf, user_buf, count)) 3167 return -EFAULT; 3168 3169 mode = sysfs_match_string(modes, buf); 3170 if (mode < 0) 3171 return mode; 3172 3173 if (mode < IWL_RESET_MODE_REMOVE_ONLY) { 3174 if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status)) 3175 return -EINVAL; 3176 if (mode == IWL_RESET_MODE_TOP_RESET) { 3177 if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_SC) 3178 return -EINVAL; 3179 trans->request_top_reset = 1; 3180 } 3181 iwl_op_mode_nic_error(trans->op_mode, IWL_ERR_TYPE_DEBUGFS); 3182 iwl_trans_schedule_reset(trans, IWL_ERR_TYPE_DEBUGFS); 3183 return count; 3184 } 3185 3186 iwl_trans_pcie_reset(trans, mode); 3187 3188 return count; 3189 } 3190 3191 DEBUGFS_READ_WRITE_FILE_OPS(interrupt); 3192 DEBUGFS_READ_FILE_OPS(fh_reg); 3193 DEBUGFS_READ_FILE_OPS(rx_queue); 3194 DEBUGFS_WRITE_FILE_OPS(csr); 3195 DEBUGFS_READ_WRITE_FILE_OPS(rfkill); 3196 DEBUGFS_READ_FILE_OPS(rf); 3197 DEBUGFS_WRITE_FILE_OPS(reset); 3198 3199 static const struct file_operations iwl_dbgfs_tx_queue_ops = { 3200 .owner = THIS_MODULE, 3201 .open = iwl_dbgfs_tx_queue_open, 3202 .read = seq_read, 3203 .llseek = seq_lseek, 3204 .release = seq_release_private, 3205 }; 3206 3207 static const struct file_operations iwl_dbgfs_monitor_data_ops = { 3208 .read = iwl_dbgfs_monitor_data_read, 3209 .open = iwl_dbgfs_monitor_data_open, 3210 .release = iwl_dbgfs_monitor_data_release, 3211 }; 3212 3213 /* Create the debugfs files and directories */ 3214 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) 3215 { 3216 struct dentry *dir = trans->dbgfs_dir; 3217 3218 DEBUGFS_ADD_FILE(rx_queue, dir, 0400); 3219 DEBUGFS_ADD_FILE(tx_queue, dir, 0400); 3220 DEBUGFS_ADD_FILE(interrupt, dir, 0600); 3221 DEBUGFS_ADD_FILE(csr, dir, 0200); 3222 DEBUGFS_ADD_FILE(fh_reg, dir, 0400); 3223 DEBUGFS_ADD_FILE(rfkill, dir, 0600); 3224 DEBUGFS_ADD_FILE(monitor_data, dir, 0400); 3225 DEBUGFS_ADD_FILE(rf, dir, 0400); 3226 DEBUGFS_ADD_FILE(reset, dir, 0200); 3227 } 3228 3229 void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans) 3230 { 3231 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3232 struct cont_rec *data = &trans_pcie->fw_mon_data; 3233 3234 mutex_lock(&data->mutex); 3235 data->state = IWL_FW_MON_DBGFS_STATE_DISABLED; 3236 mutex_unlock(&data->mutex); 3237 } 3238 #endif /*CONFIG_IWLWIFI_DEBUGFS */ 3239 3240 static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd) 3241 { 3242 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3243 u32 cmdlen = 0; 3244 int i; 3245 3246 for (i = 0; i < trans_pcie->txqs.tfd.max_tbs; i++) 3247 cmdlen += iwl_txq_gen1_tfd_tb_get_len(trans, tfd, i); 3248 3249 return cmdlen; 3250 } 3251 3252 static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans, 3253 struct iwl_fw_error_dump_data **data, 3254 int allocated_rb_nums) 3255 { 3256 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3257 int max_len = trans_pcie->rx_buf_bytes; 3258 /* Dump RBs is supported only for pre-9000 devices (1 queue) */ 3259 struct iwl_rxq *rxq = &trans_pcie->rxq[0]; 3260 u32 i, r, j, rb_len = 0; 3261 3262 spin_lock_bh(&rxq->lock); 3263 3264 r = iwl_get_closed_rb_stts(trans, rxq); 3265 3266 for (i = rxq->read, j = 0; 3267 i != r && j < allocated_rb_nums; 3268 i = (i + 1) & RX_QUEUE_MASK, j++) { 3269 struct iwl_rx_mem_buffer *rxb = rxq->queue[i]; 3270 struct iwl_fw_error_dump_rb *rb; 3271 3272 dma_sync_single_for_cpu(trans->dev, rxb->page_dma, 3273 max_len, DMA_FROM_DEVICE); 3274 3275 rb_len += sizeof(**data) + sizeof(*rb) + max_len; 3276 3277 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RB); 3278 (*data)->len = cpu_to_le32(sizeof(*rb) + max_len); 3279 rb = (void *)(*data)->data; 3280 rb->index = cpu_to_le32(i); 3281 memcpy(rb->data, page_address(rxb->page), max_len); 3282 3283 *data = iwl_fw_error_next_data(*data); 3284 } 3285 3286 spin_unlock_bh(&rxq->lock); 3287 3288 return rb_len; 3289 } 3290 #define IWL_CSR_TO_DUMP (0x250) 3291 3292 static u32 iwl_trans_pcie_dump_csr(struct iwl_trans *trans, 3293 struct iwl_fw_error_dump_data **data) 3294 { 3295 u32 csr_len = sizeof(**data) + IWL_CSR_TO_DUMP; 3296 __le32 *val; 3297 int i; 3298 3299 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_CSR); 3300 (*data)->len = cpu_to_le32(IWL_CSR_TO_DUMP); 3301 val = (void *)(*data)->data; 3302 3303 for (i = 0; i < IWL_CSR_TO_DUMP; i += 4) 3304 *val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i)); 3305 3306 *data = iwl_fw_error_next_data(*data); 3307 3308 return csr_len; 3309 } 3310 3311 static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans, 3312 struct iwl_fw_error_dump_data **data) 3313 { 3314 u32 fh_regs_len = FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND; 3315 __le32 *val; 3316 int i; 3317 3318 if (!iwl_trans_grab_nic_access(trans)) 3319 return 0; 3320 3321 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FH_REGS); 3322 (*data)->len = cpu_to_le32(fh_regs_len); 3323 val = (void *)(*data)->data; 3324 3325 if (!trans->mac_cfg->gen2) 3326 for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND; 3327 i += sizeof(u32)) 3328 *val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i)); 3329 else 3330 for (i = iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2); 3331 i < iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2); 3332 i += sizeof(u32)) 3333 *val++ = cpu_to_le32(iwl_trans_pcie_read_prph(trans, 3334 i)); 3335 3336 iwl_trans_release_nic_access(trans); 3337 3338 *data = iwl_fw_error_next_data(*data); 3339 3340 return sizeof(**data) + fh_regs_len; 3341 } 3342 3343 static u32 3344 iwl_trans_pci_dump_marbh_monitor(struct iwl_trans *trans, 3345 struct iwl_fw_error_dump_fw_mon *fw_mon_data, 3346 u32 monitor_len) 3347 { 3348 u32 buf_size_in_dwords = (monitor_len >> 2); 3349 u32 *buffer = (u32 *)fw_mon_data->data; 3350 u32 i; 3351 3352 if (!iwl_trans_grab_nic_access(trans)) 3353 return 0; 3354 3355 iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x1); 3356 for (i = 0; i < buf_size_in_dwords; i++) 3357 buffer[i] = iwl_read_umac_prph_no_grab(trans, 3358 MON_DMARB_RD_DATA_ADDR); 3359 iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x0); 3360 3361 iwl_trans_release_nic_access(trans); 3362 3363 return monitor_len; 3364 } 3365 3366 static void 3367 iwl_trans_pcie_dump_pointers(struct iwl_trans *trans, 3368 struct iwl_fw_error_dump_fw_mon *fw_mon_data) 3369 { 3370 u32 base, base_high, write_ptr, write_ptr_val, wrap_cnt; 3371 3372 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 3373 base = DBGC_CUR_DBGBUF_BASE_ADDR_LSB; 3374 base_high = DBGC_CUR_DBGBUF_BASE_ADDR_MSB; 3375 write_ptr = DBGC_CUR_DBGBUF_STATUS; 3376 wrap_cnt = DBGC_DBGBUF_WRAP_AROUND; 3377 } else if (trans->dbg.dest_tlv) { 3378 write_ptr = le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg); 3379 wrap_cnt = le32_to_cpu(trans->dbg.dest_tlv->wrap_count); 3380 base = le32_to_cpu(trans->dbg.dest_tlv->base_reg); 3381 } else { 3382 base = MON_BUFF_BASE_ADDR; 3383 write_ptr = MON_BUFF_WRPTR; 3384 wrap_cnt = MON_BUFF_CYCLE_CNT; 3385 } 3386 3387 write_ptr_val = iwl_read_prph(trans, write_ptr); 3388 fw_mon_data->fw_mon_cycle_cnt = 3389 cpu_to_le32(iwl_read_prph(trans, wrap_cnt)); 3390 fw_mon_data->fw_mon_base_ptr = 3391 cpu_to_le32(iwl_read_prph(trans, base)); 3392 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { 3393 fw_mon_data->fw_mon_base_high_ptr = 3394 cpu_to_le32(iwl_read_prph(trans, base_high)); 3395 write_ptr_val &= DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK; 3396 /* convert wrtPtr to DWs, to align with all HWs */ 3397 write_ptr_val >>= 2; 3398 } 3399 fw_mon_data->fw_mon_wr_ptr = cpu_to_le32(write_ptr_val); 3400 } 3401 3402 static u32 3403 iwl_trans_pcie_dump_monitor(struct iwl_trans *trans, 3404 struct iwl_fw_error_dump_data **data, 3405 u32 monitor_len) 3406 { 3407 struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon; 3408 u32 len = 0; 3409 3410 if (trans->dbg.dest_tlv || 3411 (fw_mon->size && 3412 (trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_7000 || 3413 trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))) { 3414 struct iwl_fw_error_dump_fw_mon *fw_mon_data; 3415 3416 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR); 3417 fw_mon_data = (void *)(*data)->data; 3418 3419 iwl_trans_pcie_dump_pointers(trans, fw_mon_data); 3420 3421 len += sizeof(**data) + sizeof(*fw_mon_data); 3422 if (fw_mon->size) { 3423 memcpy(fw_mon_data->data, fw_mon->block, fw_mon->size); 3424 monitor_len = fw_mon->size; 3425 } else if (trans->dbg.dest_tlv->monitor_mode == SMEM_MODE) { 3426 u32 base = le32_to_cpu(fw_mon_data->fw_mon_base_ptr); 3427 /* 3428 * Update pointers to reflect actual values after 3429 * shifting 3430 */ 3431 if (trans->dbg.dest_tlv->version) { 3432 base = (iwl_read_prph(trans, base) & 3433 IWL_LDBG_M2S_BUF_BA_MSK) << 3434 trans->dbg.dest_tlv->base_shift; 3435 base *= IWL_M2S_UNIT_SIZE; 3436 base += trans->mac_cfg->base->smem_offset; 3437 } else { 3438 base = iwl_read_prph(trans, base) << 3439 trans->dbg.dest_tlv->base_shift; 3440 } 3441 3442 iwl_trans_pcie_read_mem(trans, base, fw_mon_data->data, 3443 monitor_len / sizeof(u32)); 3444 } else if (trans->dbg.dest_tlv->monitor_mode == MARBH_MODE) { 3445 monitor_len = 3446 iwl_trans_pci_dump_marbh_monitor(trans, 3447 fw_mon_data, 3448 monitor_len); 3449 } else { 3450 /* Didn't match anything - output no monitor data */ 3451 monitor_len = 0; 3452 } 3453 3454 len += monitor_len; 3455 (*data)->len = cpu_to_le32(monitor_len + sizeof(*fw_mon_data)); 3456 } 3457 3458 return len; 3459 } 3460 3461 static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, u32 *len) 3462 { 3463 if (trans->dbg.fw_mon.size) { 3464 *len += sizeof(struct iwl_fw_error_dump_data) + 3465 sizeof(struct iwl_fw_error_dump_fw_mon) + 3466 trans->dbg.fw_mon.size; 3467 return trans->dbg.fw_mon.size; 3468 } else if (trans->dbg.dest_tlv) { 3469 u32 base, end, cfg_reg, monitor_len; 3470 3471 if (trans->dbg.dest_tlv->version == 1) { 3472 cfg_reg = le32_to_cpu(trans->dbg.dest_tlv->base_reg); 3473 cfg_reg = iwl_read_prph(trans, cfg_reg); 3474 base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) << 3475 trans->dbg.dest_tlv->base_shift; 3476 base *= IWL_M2S_UNIT_SIZE; 3477 base += trans->mac_cfg->base->smem_offset; 3478 3479 monitor_len = 3480 (cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >> 3481 trans->dbg.dest_tlv->end_shift; 3482 monitor_len *= IWL_M2S_UNIT_SIZE; 3483 } else { 3484 base = le32_to_cpu(trans->dbg.dest_tlv->base_reg); 3485 end = le32_to_cpu(trans->dbg.dest_tlv->end_reg); 3486 3487 base = iwl_read_prph(trans, base) << 3488 trans->dbg.dest_tlv->base_shift; 3489 end = iwl_read_prph(trans, end) << 3490 trans->dbg.dest_tlv->end_shift; 3491 3492 /* Make "end" point to the actual end */ 3493 if (trans->mac_cfg->device_family >= 3494 IWL_DEVICE_FAMILY_8000 || 3495 trans->dbg.dest_tlv->monitor_mode == MARBH_MODE) 3496 end += (1 << trans->dbg.dest_tlv->end_shift); 3497 monitor_len = end - base; 3498 } 3499 *len += sizeof(struct iwl_fw_error_dump_data) + 3500 sizeof(struct iwl_fw_error_dump_fw_mon) + 3501 monitor_len; 3502 return monitor_len; 3503 } 3504 return 0; 3505 } 3506 3507 struct iwl_trans_dump_data * 3508 iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask, 3509 const struct iwl_dump_sanitize_ops *sanitize_ops, 3510 void *sanitize_ctx) 3511 { 3512 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3513 struct iwl_fw_error_dump_data *data; 3514 struct iwl_txq *cmdq = trans_pcie->txqs.txq[trans->conf.cmd_queue]; 3515 struct iwl_fw_error_dump_txcmd *txcmd; 3516 struct iwl_trans_dump_data *dump_data; 3517 u32 len, num_rbs = 0, monitor_len = 0; 3518 int i, ptr; 3519 bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) && 3520 !trans->mac_cfg->mq_rx_supported && 3521 dump_mask & BIT(IWL_FW_ERROR_DUMP_RB); 3522 3523 if (!dump_mask) 3524 return NULL; 3525 3526 /* transport dump header */ 3527 len = sizeof(*dump_data); 3528 3529 /* host commands */ 3530 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq) 3531 len += sizeof(*data) + 3532 cmdq->n_window * (sizeof(*txcmd) + 3533 TFD_MAX_PAYLOAD_SIZE); 3534 3535 /* FW monitor */ 3536 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR)) 3537 monitor_len = iwl_trans_get_fw_monitor_len(trans, &len); 3538 3539 /* CSR registers */ 3540 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR)) 3541 len += sizeof(*data) + IWL_CSR_TO_DUMP; 3542 3543 /* FH registers */ 3544 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) { 3545 if (trans->mac_cfg->gen2) 3546 len += sizeof(*data) + 3547 (iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2) - 3548 iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2)); 3549 else 3550 len += sizeof(*data) + 3551 (FH_MEM_UPPER_BOUND - 3552 FH_MEM_LOWER_BOUND); 3553 } 3554 3555 if (dump_rbs) { 3556 /* Dump RBs is supported only for pre-9000 devices (1 queue) */ 3557 struct iwl_rxq *rxq = &trans_pcie->rxq[0]; 3558 /* RBs */ 3559 spin_lock_bh(&rxq->lock); 3560 num_rbs = iwl_get_closed_rb_stts(trans, rxq); 3561 num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK; 3562 spin_unlock_bh(&rxq->lock); 3563 3564 len += num_rbs * (sizeof(*data) + 3565 sizeof(struct iwl_fw_error_dump_rb) + 3566 (PAGE_SIZE << trans_pcie->rx_page_order)); 3567 } 3568 3569 /* Paged memory for gen2 HW */ 3570 if (trans->mac_cfg->gen2 && dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) 3571 for (i = 0; i < trans->init_dram.paging_cnt; i++) 3572 len += sizeof(*data) + 3573 sizeof(struct iwl_fw_error_dump_paging) + 3574 trans->init_dram.paging[i].size; 3575 3576 dump_data = vzalloc(len); 3577 if (!dump_data) 3578 return NULL; 3579 3580 len = 0; 3581 data = (void *)dump_data->data; 3582 3583 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq) { 3584 u16 tfd_size = trans_pcie->txqs.tfd.size; 3585 3586 data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD); 3587 txcmd = (void *)data->data; 3588 spin_lock_bh(&cmdq->lock); 3589 ptr = cmdq->write_ptr; 3590 for (i = 0; i < cmdq->n_window; i++) { 3591 u8 idx = iwl_txq_get_cmd_index(cmdq, ptr); 3592 u8 tfdidx; 3593 u32 caplen, cmdlen; 3594 3595 if (trans->mac_cfg->gen2) 3596 tfdidx = idx; 3597 else 3598 tfdidx = ptr; 3599 3600 cmdlen = iwl_trans_pcie_get_cmdlen(trans, 3601 (u8 *)cmdq->tfds + 3602 tfd_size * tfdidx); 3603 caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen); 3604 3605 if (cmdlen) { 3606 len += sizeof(*txcmd) + caplen; 3607 txcmd->cmdlen = cpu_to_le32(cmdlen); 3608 txcmd->caplen = cpu_to_le32(caplen); 3609 memcpy(txcmd->data, cmdq->entries[idx].cmd, 3610 caplen); 3611 if (sanitize_ops && sanitize_ops->frob_hcmd) 3612 sanitize_ops->frob_hcmd(sanitize_ctx, 3613 txcmd->data, 3614 caplen); 3615 txcmd = (void *)((u8 *)txcmd->data + caplen); 3616 } 3617 3618 ptr = iwl_txq_dec_wrap(trans, ptr); 3619 } 3620 spin_unlock_bh(&cmdq->lock); 3621 3622 data->len = cpu_to_le32(len); 3623 len += sizeof(*data); 3624 data = iwl_fw_error_next_data(data); 3625 } 3626 3627 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR)) 3628 len += iwl_trans_pcie_dump_csr(trans, &data); 3629 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) 3630 len += iwl_trans_pcie_fh_regs_dump(trans, &data); 3631 if (dump_rbs) 3632 len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs); 3633 3634 /* Paged memory for gen2 HW */ 3635 if (trans->mac_cfg->gen2 && 3636 dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) { 3637 for (i = 0; i < trans->init_dram.paging_cnt; i++) { 3638 struct iwl_fw_error_dump_paging *paging; 3639 u32 page_len = trans->init_dram.paging[i].size; 3640 3641 data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING); 3642 data->len = cpu_to_le32(sizeof(*paging) + page_len); 3643 paging = (void *)data->data; 3644 paging->index = cpu_to_le32(i); 3645 memcpy(paging->data, 3646 trans->init_dram.paging[i].block, page_len); 3647 data = iwl_fw_error_next_data(data); 3648 3649 len += sizeof(*data) + sizeof(*paging) + page_len; 3650 } 3651 } 3652 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR)) 3653 len += iwl_trans_pcie_dump_monitor(trans, &data, monitor_len); 3654 3655 dump_data->len = len; 3656 3657 return dump_data; 3658 } 3659 3660 void iwl_trans_pci_interrupts(struct iwl_trans *trans, bool enable) 3661 { 3662 if (enable) 3663 iwl_enable_interrupts(trans); 3664 else 3665 iwl_disable_interrupts(trans); 3666 } 3667 3668 void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans) 3669 { 3670 u32 inta_addr, sw_err_bit; 3671 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3672 3673 if (trans_pcie->msix_enabled) { 3674 inta_addr = CSR_MSIX_HW_INT_CAUSES_AD; 3675 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) 3676 sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ; 3677 else 3678 sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR; 3679 } else { 3680 inta_addr = CSR_INT; 3681 sw_err_bit = CSR_INT_BIT_SW_ERR; 3682 } 3683 3684 iwl_trans_sync_nmi_with_addr(trans, inta_addr, sw_err_bit); 3685 } 3686 3687 static int iwl_trans_pcie_set_txcmd_info(const struct iwl_mac_cfg *mac_cfg, 3688 unsigned int *txcmd_size, 3689 unsigned int *txcmd_align) 3690 { 3691 if (!mac_cfg->gen2) { 3692 *txcmd_size = sizeof(struct iwl_tx_cmd_v6); 3693 *txcmd_align = sizeof(void *); 3694 } else if (mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { 3695 *txcmd_size = sizeof(struct iwl_tx_cmd_v9); 3696 *txcmd_align = 64; 3697 } else { 3698 *txcmd_size = sizeof(struct iwl_tx_cmd); 3699 *txcmd_align = 128; 3700 } 3701 3702 *txcmd_size += sizeof(struct iwl_cmd_header); 3703 *txcmd_size += 36; /* biggest possible 802.11 header */ 3704 3705 /* Ensure device TX cmd cannot reach/cross a page boundary in gen2 */ 3706 if (WARN_ON((mac_cfg->gen2 && *txcmd_size >= *txcmd_align))) 3707 return -EINVAL; 3708 3709 return 0; 3710 } 3711 3712 static struct iwl_trans * 3713 iwl_trans_pcie_alloc(struct pci_dev *pdev, 3714 const struct iwl_mac_cfg *mac_cfg, 3715 struct iwl_trans_info *info, u8 __iomem *hw_base) 3716 { 3717 struct iwl_trans_pcie *trans_pcie, **priv; 3718 unsigned int txcmd_size, txcmd_align; 3719 struct iwl_trans *trans; 3720 unsigned int bc_tbl_n_entries; 3721 int ret, addr_size; 3722 3723 ret = iwl_trans_pcie_set_txcmd_info(mac_cfg, &txcmd_size, 3724 &txcmd_align); 3725 if (ret) 3726 return ERR_PTR(ret); 3727 3728 trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie), &pdev->dev, 3729 mac_cfg, txcmd_size, txcmd_align); 3730 if (!trans) 3731 return ERR_PTR(-ENOMEM); 3732 3733 trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3734 3735 trans_pcie->hw_base = hw_base; 3736 3737 /* Initialize the wait queue for commands */ 3738 init_waitqueue_head(&trans_pcie->wait_command_queue); 3739 3740 if (trans->mac_cfg->gen2) { 3741 trans_pcie->txqs.tfd.addr_size = 64; 3742 trans_pcie->txqs.tfd.max_tbs = IWL_TFH_NUM_TBS; 3743 trans_pcie->txqs.tfd.size = sizeof(struct iwl_tfh_tfd); 3744 } else { 3745 trans_pcie->txqs.tfd.addr_size = 36; 3746 trans_pcie->txqs.tfd.max_tbs = IWL_NUM_OF_TBS; 3747 trans_pcie->txqs.tfd.size = sizeof(struct iwl_tfd); 3748 } 3749 3750 trans_pcie->supported_dma_mask = (u32)DMA_BIT_MASK(12); 3751 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) 3752 trans_pcie->supported_dma_mask = (u32)DMA_BIT_MASK(11); 3753 3754 info->max_skb_frags = IWL_TRANS_PCIE_MAX_FRAGS(trans_pcie); 3755 3756 trans_pcie->txqs.tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page); 3757 if (!trans_pcie->txqs.tso_hdr_page) { 3758 ret = -ENOMEM; 3759 goto out_free_trans; 3760 } 3761 3762 if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) 3763 bc_tbl_n_entries = TFD_QUEUE_BC_SIZE_BZ; 3764 else if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) 3765 bc_tbl_n_entries = TFD_QUEUE_BC_SIZE_AX210; 3766 else 3767 bc_tbl_n_entries = TFD_QUEUE_BC_SIZE; 3768 3769 trans_pcie->txqs.bc_tbl_size = 3770 sizeof(struct iwl_bc_tbl_entry) * bc_tbl_n_entries; 3771 /* 3772 * For gen2 devices, we use a single allocation for each byte-count 3773 * table, but they're pretty small (1k) so use a DMA pool that we 3774 * allocate here. 3775 */ 3776 if (trans->mac_cfg->gen2) { 3777 trans_pcie->txqs.bc_pool = 3778 dmam_pool_create("iwlwifi:bc", trans->dev, 3779 trans_pcie->txqs.bc_tbl_size, 3780 256, 0); 3781 if (!trans_pcie->txqs.bc_pool) { 3782 ret = -ENOMEM; 3783 goto out_free_tso; 3784 } 3785 } 3786 3787 /* Some things must not change even if the config does */ 3788 WARN_ON(trans_pcie->txqs.tfd.addr_size != 3789 (trans->mac_cfg->gen2 ? 64 : 36)); 3790 3791 /* Initialize NAPI here - it should be before registering to mac80211 3792 * in the opmode but after the HW struct is allocated. 3793 */ 3794 trans_pcie->napi_dev = alloc_netdev_dummy(sizeof(struct iwl_trans_pcie *)); 3795 if (!trans_pcie->napi_dev) { 3796 ret = -ENOMEM; 3797 goto out_free_tso; 3798 } 3799 /* The private struct in netdev is a pointer to struct iwl_trans_pcie */ 3800 priv = netdev_priv(trans_pcie->napi_dev); 3801 *priv = trans_pcie; 3802 3803 trans_pcie->trans = trans; 3804 trans_pcie->opmode_down = true; 3805 spin_lock_init(&trans_pcie->irq_lock); 3806 spin_lock_init(&trans_pcie->reg_lock); 3807 spin_lock_init(&trans_pcie->alloc_page_lock); 3808 mutex_init(&trans_pcie->mutex); 3809 init_waitqueue_head(&trans_pcie->ucode_write_waitq); 3810 init_waitqueue_head(&trans_pcie->fw_reset_waitq); 3811 init_waitqueue_head(&trans_pcie->imr_waitq); 3812 3813 trans_pcie->rba.alloc_wq = alloc_workqueue("rb_allocator", 3814 WQ_HIGHPRI | WQ_UNBOUND, 0); 3815 if (!trans_pcie->rba.alloc_wq) { 3816 ret = -ENOMEM; 3817 goto out_free_ndev; 3818 } 3819 INIT_WORK(&trans_pcie->rba.rx_alloc, iwl_pcie_rx_allocator_work); 3820 3821 trans_pcie->debug_rfkill = -1; 3822 3823 if (!mac_cfg->base->pcie_l1_allowed) { 3824 /* 3825 * W/A - seems to solve weird behavior. We need to remove this 3826 * if we don't want to stay in L1 all the time. This wastes a 3827 * lot of power. 3828 */ 3829 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | 3830 PCIE_LINK_STATE_L1 | 3831 PCIE_LINK_STATE_CLKPM); 3832 } 3833 3834 addr_size = trans_pcie->txqs.tfd.addr_size; 3835 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_size)); 3836 if (ret) { 3837 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3838 /* both attempts failed: */ 3839 if (ret) { 3840 dev_err(&pdev->dev, "No suitable DMA available\n"); 3841 goto out_no_pci; 3842 } 3843 } 3844 3845 /* We disable the RETRY_TIMEOUT register (0x41) to keep 3846 * PCI Tx retries from interfering with C3 CPU state */ 3847 pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00); 3848 3849 trans_pcie->pci_dev = pdev; 3850 iwl_disable_interrupts(trans); 3851 3852 /* 3853 * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have 3854 * changed, and now the revision step also includes bit 0-1 (no more 3855 * "dash" value). To keep hw_rev backwards compatible - we'll store it 3856 * in the old format. 3857 */ 3858 if (mac_cfg->device_family >= IWL_DEVICE_FAMILY_8000) 3859 info->hw_rev_step = info->hw_rev & 0xF; 3860 else 3861 info->hw_rev_step = (info->hw_rev & 0xC) >> 2; 3862 3863 IWL_DEBUG_INFO(trans, "HW REV: 0x%0x\n", info->hw_rev); 3864 3865 iwl_pcie_set_interrupt_capa(pdev, trans, mac_cfg, info); 3866 3867 init_waitqueue_head(&trans_pcie->sx_waitq); 3868 3869 ret = iwl_pcie_alloc_invalid_tx_cmd(trans); 3870 if (ret) 3871 goto out_no_pci; 3872 3873 if (trans_pcie->msix_enabled) { 3874 ret = iwl_pcie_init_msix_handler(pdev, trans_pcie, info); 3875 if (ret) 3876 goto out_no_pci; 3877 } else { 3878 ret = iwl_pcie_alloc_ict(trans); 3879 if (ret) 3880 goto out_no_pci; 3881 3882 ret = devm_request_threaded_irq(&pdev->dev, pdev->irq, 3883 iwl_pcie_isr, 3884 iwl_pcie_irq_handler, 3885 IRQF_SHARED, DRV_NAME, trans); 3886 if (ret) { 3887 IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq); 3888 goto out_free_ict; 3889 } 3890 } 3891 3892 #ifdef CONFIG_IWLWIFI_DEBUGFS 3893 trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED; 3894 mutex_init(&trans_pcie->fw_mon_data.mutex); 3895 #endif 3896 3897 iwl_dbg_tlv_init(trans); 3898 3899 return trans; 3900 3901 out_free_ict: 3902 iwl_pcie_free_ict(trans); 3903 out_no_pci: 3904 destroy_workqueue(trans_pcie->rba.alloc_wq); 3905 out_free_ndev: 3906 free_netdev(trans_pcie->napi_dev); 3907 out_free_tso: 3908 free_percpu(trans_pcie->txqs.tso_hdr_page); 3909 out_free_trans: 3910 iwl_trans_free(trans); 3911 return ERR_PTR(ret); 3912 } 3913 3914 void iwl_trans_pcie_copy_imr_fh(struct iwl_trans *trans, 3915 u32 dst_addr, u64 src_addr, u32 byte_cnt) 3916 { 3917 iwl_write_prph(trans, IMR_UREG_CHICK, 3918 iwl_read_prph(trans, IMR_UREG_CHICK) | 3919 IMR_UREG_CHICK_HALT_UMAC_PERMANENTLY_MSK); 3920 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_SRAM_ADDR, dst_addr); 3921 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_DRAM_ADDR_LSB, 3922 (u32)(src_addr & 0xFFFFFFFF)); 3923 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_DRAM_ADDR_MSB, 3924 iwl_get_dma_hi_addr(src_addr)); 3925 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_BC, byte_cnt); 3926 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_CTRL, 3927 IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_IRQ_TARGET_POS | 3928 IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_DMA_EN_POS | 3929 IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_RS_MSK); 3930 } 3931 3932 int iwl_trans_pcie_copy_imr(struct iwl_trans *trans, 3933 u32 dst_addr, u64 src_addr, u32 byte_cnt) 3934 { 3935 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 3936 int ret = -1; 3937 3938 trans_pcie->imr_status = IMR_D2S_REQUESTED; 3939 iwl_trans_pcie_copy_imr_fh(trans, dst_addr, src_addr, byte_cnt); 3940 ret = wait_event_timeout(trans_pcie->imr_waitq, 3941 trans_pcie->imr_status != 3942 IMR_D2S_REQUESTED, 5 * HZ); 3943 if (!ret || trans_pcie->imr_status == IMR_D2S_ERROR) { 3944 IWL_ERR(trans, "Failed to copy IMR Memory chunk!\n"); 3945 iwl_trans_pcie_dump_regs(trans, trans_pcie->pci_dev); 3946 return -ETIMEDOUT; 3947 } 3948 trans_pcie->imr_status = IMR_D2S_IDLE; 3949 return 0; 3950 } 3951 3952 /* 3953 * Read rf id and cdb info from prph register and store it 3954 */ 3955 static void get_crf_id(struct iwl_trans *iwl_trans, 3956 struct iwl_trans_info *info) 3957 { 3958 u32 sd_reg_ver_addr; 3959 u32 hw_wfpm_id; 3960 u32 val = 0; 3961 u8 step; 3962 3963 if (iwl_trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) 3964 sd_reg_ver_addr = SD_REG_VER_GEN2; 3965 else 3966 sd_reg_ver_addr = SD_REG_VER; 3967 3968 /* Enable access to peripheral registers */ 3969 val = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG); 3970 val |= WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK; 3971 iwl_write_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG, val); 3972 3973 /* Read crf info */ 3974 info->hw_crf_id = iwl_read_prph_no_grab(iwl_trans, sd_reg_ver_addr); 3975 3976 /* Read cnv info */ 3977 info->hw_cnv_id = iwl_read_prph_no_grab(iwl_trans, CNVI_AUX_MISC_CHIP); 3978 3979 /* For BZ-W, take B step also when A step is indicated */ 3980 if (CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ_W) 3981 step = SILICON_B_STEP; 3982 3983 /* In BZ, the MAC step must be read from the CNVI aux register */ 3984 if (CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ) { 3985 step = CNVI_AUX_MISC_CHIP_MAC_STEP(info->hw_cnv_id); 3986 3987 /* For BZ-U, take B step also when A step is indicated */ 3988 if ((CNVI_AUX_MISC_CHIP_PROD_TYPE(info->hw_cnv_id) == 3989 CNVI_AUX_MISC_CHIP_PROD_TYPE_BZ_U) && 3990 step == SILICON_A_STEP) 3991 step = SILICON_B_STEP; 3992 } 3993 3994 if (CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ || 3995 CSR_HW_REV_TYPE(info->hw_rev) == IWL_CFG_MAC_TYPE_BZ_W) { 3996 info->hw_rev_step = step; 3997 info->hw_rev |= step; 3998 } 3999 4000 /* Read cdb info (also contains the jacket info if needed in the future */ 4001 hw_wfpm_id = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_OTP_CFG1_ADDR); 4002 IWL_INFO(iwl_trans, "Detected crf-id 0x%x, cnv-id 0x%x wfpm id 0x%x\n", 4003 info->hw_crf_id, info->hw_cnv_id, hw_wfpm_id); 4004 } 4005 4006 /* 4007 * In case that there is no OTP on the NIC, map the rf id and cdb info 4008 * from the prph registers. 4009 */ 4010 static int map_crf_id(struct iwl_trans *iwl_trans, 4011 struct iwl_trans_info *info) 4012 { 4013 int ret = 0; 4014 u32 val = info->hw_crf_id; 4015 u32 step_id = REG_CRF_ID_STEP(val); 4016 u32 slave_id = REG_CRF_ID_SLAVE(val); 4017 u32 jacket_id_cnv = REG_CRF_ID_SLAVE(info->hw_cnv_id); 4018 u32 hw_wfpm_id = iwl_read_umac_prph_no_grab(iwl_trans, 4019 WFPM_OTP_CFG1_ADDR); 4020 u32 jacket_id_wfpm = WFPM_OTP_CFG1_IS_JACKET(hw_wfpm_id); 4021 u32 cdb_id_wfpm = WFPM_OTP_CFG1_IS_CDB(hw_wfpm_id); 4022 4023 /* Map between crf id to rf id */ 4024 switch (REG_CRF_ID_TYPE(val)) { 4025 case REG_CRF_ID_TYPE_JF_1: 4026 info->hw_rf_id = (IWL_CFG_RF_TYPE_JF1 << 12); 4027 break; 4028 case REG_CRF_ID_TYPE_JF_2: 4029 info->hw_rf_id = (IWL_CFG_RF_TYPE_JF2 << 12); 4030 break; 4031 case REG_CRF_ID_TYPE_HR_NONE_CDB_1X1: 4032 info->hw_rf_id = (IWL_CFG_RF_TYPE_HR1 << 12); 4033 break; 4034 case REG_CRF_ID_TYPE_HR_NONE_CDB: 4035 info->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12); 4036 break; 4037 case REG_CRF_ID_TYPE_HR_CDB: 4038 info->hw_rf_id = (IWL_CFG_RF_TYPE_HR2 << 12); 4039 break; 4040 case REG_CRF_ID_TYPE_GF: 4041 info->hw_rf_id = (IWL_CFG_RF_TYPE_GF << 12); 4042 break; 4043 case REG_CRF_ID_TYPE_FM: 4044 info->hw_rf_id = (IWL_CFG_RF_TYPE_FM << 12); 4045 break; 4046 case REG_CRF_ID_TYPE_WHP: 4047 info->hw_rf_id = (IWL_CFG_RF_TYPE_WH << 12); 4048 break; 4049 case REG_CRF_ID_TYPE_PE: 4050 info->hw_rf_id = (IWL_CFG_RF_TYPE_PE << 12); 4051 break; 4052 default: 4053 ret = -EIO; 4054 IWL_ERR(iwl_trans, 4055 "Can't find a correct rfid for crf id 0x%x\n", 4056 REG_CRF_ID_TYPE(val)); 4057 goto out; 4058 } 4059 4060 /* Set Step-id */ 4061 info->hw_rf_id |= (step_id << 8); 4062 4063 /* Set CDB capabilities */ 4064 if (cdb_id_wfpm || slave_id) { 4065 info->hw_rf_id += BIT(28); 4066 IWL_INFO(iwl_trans, "Adding cdb to rf id\n"); 4067 } 4068 4069 /* Set Jacket capabilities */ 4070 if (jacket_id_wfpm || jacket_id_cnv) { 4071 info->hw_rf_id += BIT(29); 4072 IWL_INFO(iwl_trans, "Adding jacket to rf id\n"); 4073 } 4074 4075 IWL_INFO(iwl_trans, 4076 "Detected rf-type 0x%x step-id 0x%x slave-id 0x%x from crf id 0x%x\n", 4077 REG_CRF_ID_TYPE(val), step_id, slave_id, info->hw_rf_id); 4078 IWL_INFO(iwl_trans, 4079 "Detected cdb-id 0x%x jacket-id 0x%x from wfpm id 0x%x\n", 4080 cdb_id_wfpm, jacket_id_wfpm, hw_wfpm_id); 4081 IWL_INFO(iwl_trans, "Detected jacket-id 0x%x from cnvi id 0x%x\n", 4082 jacket_id_cnv, info->hw_cnv_id); 4083 4084 out: 4085 return ret; 4086 } 4087 4088 static void iwl_pcie_recheck_me_status(struct work_struct *wk) 4089 { 4090 struct iwl_trans_pcie *trans_pcie = container_of(wk, 4091 typeof(*trans_pcie), 4092 me_recheck_wk.work); 4093 u32 val; 4094 4095 val = iwl_read32(trans_pcie->trans, CSR_HW_IF_CONFIG_REG); 4096 trans_pcie->me_present = !!(val & CSR_HW_IF_CONFIG_REG_IAMT_UP); 4097 } 4098 4099 static void iwl_pcie_check_me_status(struct iwl_trans *trans) 4100 { 4101 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); 4102 u32 val; 4103 4104 trans_pcie->me_present = -1; 4105 4106 INIT_DELAYED_WORK(&trans_pcie->me_recheck_wk, 4107 iwl_pcie_recheck_me_status); 4108 4109 /* we don't have a good way of determining this until BZ */ 4110 if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ) 4111 return; 4112 4113 val = iwl_read_prph(trans, CNVI_SCU_REG_FOR_ECO_1); 4114 if (val & CNVI_SCU_REG_FOR_ECO_1_WIAMT_KNOWN) { 4115 trans_pcie->me_present = 4116 !!(val & CNVI_SCU_REG_FOR_ECO_1_WIAMT_PRESENT); 4117 return; 4118 } 4119 4120 val = iwl_read32(trans, CSR_HW_IF_CONFIG_REG); 4121 if (val & (CSR_HW_IF_CONFIG_REG_ME_OWN | 4122 CSR_HW_IF_CONFIG_REG_IAMT_UP)) { 4123 trans_pcie->me_present = 1; 4124 return; 4125 } 4126 4127 /* recheck again later, ME might still be initializing */ 4128 schedule_delayed_work(&trans_pcie->me_recheck_wk, HZ); 4129 } 4130 4131 int iwl_pci_gen1_2_probe(struct pci_dev *pdev, 4132 const struct pci_device_id *ent, 4133 const struct iwl_mac_cfg *mac_cfg, 4134 u8 __iomem *hw_base, u32 hw_rev) 4135 { 4136 const struct iwl_dev_info *dev_info; 4137 struct iwl_trans_info info = { 4138 .hw_id = (pdev->device << 16) + pdev->subsystem_device, 4139 .hw_rev = hw_rev, 4140 }; 4141 struct iwl_trans *iwl_trans; 4142 struct iwl_trans_pcie *trans_pcie; 4143 int ret; 4144 4145 iwl_trans = iwl_trans_pcie_alloc(pdev, mac_cfg, &info, hw_base); 4146 if (IS_ERR(iwl_trans)) 4147 return PTR_ERR(iwl_trans); 4148 4149 trans_pcie = IWL_TRANS_GET_PCIE_TRANS(iwl_trans); 4150 4151 iwl_trans_pcie_check_product_reset_status(pdev); 4152 iwl_trans_pcie_check_product_reset_mode(pdev); 4153 4154 /* set the things we know so far for the grab NIC access */ 4155 iwl_trans_set_info(iwl_trans, &info); 4156 4157 /* 4158 * Let's try to grab NIC access early here. Sometimes, NICs may 4159 * fail to initialize, and if that happens it's better if we see 4160 * issues early on (and can reprobe, per the logic inside), than 4161 * first trying to load the firmware etc. and potentially only 4162 * detecting any problems when the first interface is brought up. 4163 */ 4164 ret = iwl_pcie_prepare_card_hw(iwl_trans); 4165 if (!ret) { 4166 ret = iwl_finish_nic_init(iwl_trans); 4167 if (ret) 4168 goto out_free_trans; 4169 if (iwl_trans_grab_nic_access(iwl_trans)) { 4170 get_crf_id(iwl_trans, &info); 4171 /* all good */ 4172 iwl_trans_release_nic_access(iwl_trans); 4173 } else { 4174 ret = -EIO; 4175 goto out_free_trans; 4176 } 4177 } 4178 4179 info.hw_rf_id = iwl_read32(iwl_trans, CSR_HW_RF_ID); 4180 4181 /* 4182 * The RF_ID is set to zero in blank OTP so read version to 4183 * extract the RF_ID. 4184 * This is relevant only for family 9000 and up. 4185 */ 4186 if (iwl_trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_9000 && 4187 !CSR_HW_RFID_TYPE(info.hw_rf_id) && map_crf_id(iwl_trans, &info)) { 4188 ret = -EINVAL; 4189 goto out_free_trans; 4190 } 4191 4192 IWL_INFO(iwl_trans, "PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n", 4193 pdev->device, pdev->subsystem_device, 4194 info.hw_rev, info.hw_rf_id); 4195 4196 dev_info = iwl_pci_find_dev_info(pdev->device, pdev->subsystem_device, 4197 CSR_HW_RFID_TYPE(info.hw_rf_id), 4198 CSR_HW_RFID_IS_CDB(info.hw_rf_id), 4199 IWL_SUBDEVICE_RF_ID(pdev->subsystem_device), 4200 IWL_SUBDEVICE_BW_LIM(pdev->subsystem_device), 4201 !iwl_trans->mac_cfg->integrated); 4202 if (dev_info) { 4203 iwl_trans->cfg = dev_info->cfg; 4204 info.name = dev_info->name; 4205 } 4206 4207 #if IS_ENABLED(CONFIG_IWLMVM) 4208 4209 /* 4210 * special-case 7265D, it has the same PCI IDs. 4211 * 4212 * Note that because we already pass the cfg to the transport above, 4213 * all the parameters that the transport uses must, until that is 4214 * changed, be identical to the ones in the 7265D configuration. 4215 */ 4216 if (iwl_trans->cfg == &iwl7265_cfg && 4217 (info.hw_rev & CSR_HW_REV_TYPE_MSK) == CSR_HW_REV_TYPE_7265D) 4218 iwl_trans->cfg = &iwl7265d_cfg; 4219 #endif 4220 if (!iwl_trans->cfg) { 4221 pr_err("No config found for PCI dev %04x/%04x, rev=0x%x, rfid=0x%x\n", 4222 pdev->device, pdev->subsystem_device, 4223 info.hw_rev, info.hw_rf_id); 4224 ret = -EINVAL; 4225 goto out_free_trans; 4226 } 4227 4228 IWL_INFO(iwl_trans, "Detected %s\n", info.name); 4229 4230 if (iwl_trans->mac_cfg->mq_rx_supported) { 4231 if (WARN_ON(!iwl_trans->cfg->num_rbds)) { 4232 ret = -EINVAL; 4233 goto out_free_trans; 4234 } 4235 trans_pcie->num_rx_bufs = iwl_trans_get_num_rbds(iwl_trans); 4236 } else { 4237 trans_pcie->num_rx_bufs = RX_QUEUE_SIZE; 4238 } 4239 4240 if (!iwl_trans->mac_cfg->integrated) { 4241 u16 link_status; 4242 4243 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &link_status); 4244 4245 info.pcie_link_speed = 4246 u16_get_bits(link_status, PCI_EXP_LNKSTA_CLS); 4247 } 4248 4249 iwl_trans_set_info(iwl_trans, &info); 4250 4251 pci_set_drvdata(pdev, iwl_trans); 4252 4253 iwl_pcie_check_me_status(iwl_trans); 4254 4255 /* try to get ownership so that we'll know if we don't own it */ 4256 iwl_pcie_prepare_card_hw(iwl_trans); 4257 4258 iwl_trans->drv = iwl_drv_start(iwl_trans); 4259 4260 if (IS_ERR(iwl_trans->drv)) { 4261 ret = PTR_ERR(iwl_trans->drv); 4262 goto out_free_trans; 4263 } 4264 4265 /* register transport layer debugfs here */ 4266 iwl_trans_pcie_dbgfs_register(iwl_trans); 4267 4268 return 0; 4269 4270 out_free_trans: 4271 iwl_trans_pcie_free(iwl_trans); 4272 return ret; 4273 } 4274