1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021, MediaTek Inc. 4 * Copyright (c) 2021-2022, Intel Corporation. 5 * 6 * Authors: 7 * Haijun Liu <haijun.liu@mediatek.com> 8 * Ricardo Martinez <ricardo.martinez@linux.intel.com> 9 * Sreehari Kancharla <sreehari.kancharla@intel.com> 10 * 11 * Contributors: 12 * Amir Hanania <amir.hanania@intel.com> 13 * Andy Shevchenko <andriy.shevchenko@linux.intel.com> 14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15 * Eliot Lee <eliot.lee@intel.com> 16 * Moises Veleta <moises.veleta@intel.com> 17 */ 18 19 #include <linux/atomic.h> 20 #include <linux/bits.h> 21 #include <linux/completion.h> 22 #include <linux/device.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/gfp.h> 25 #include <linux/interrupt.h> 26 #include <linux/io.h> 27 #include <linux/iopoll.h> 28 #include <linux/jiffies.h> 29 #include <linux/list.h> 30 #include <linux/module.h> 31 #include <linux/mutex.h> 32 #include <linux/pci.h> 33 #include <linux/pm.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/spinlock.h> 36 37 #include "t7xx_mhccif.h" 38 #include "t7xx_modem_ops.h" 39 #include "t7xx_pci.h" 40 #include "t7xx_pcie_mac.h" 41 #include "t7xx_reg.h" 42 #include "t7xx_state_monitor.h" 43 #include "t7xx_port_proxy.h" 44 45 #define DRIVER_NAME "mtk_t7xx" 46 47 #define T7XX_PCI_IREG_BASE 0 48 #define T7XX_PCI_EREG_BASE 2 49 50 #define T7XX_INIT_TIMEOUT 20 51 #define PM_SLEEP_DIS_TIMEOUT_MS 20 52 #define PM_ACK_TIMEOUT_MS 1500 53 #define PM_AUTOSUSPEND_MS 5000 54 #define PM_RESOURCE_POLL_TIMEOUT_US 10000 55 #define PM_RESOURCE_POLL_STEP_US 100 56 57 static const char * const t7xx_mode_names[] = { 58 [T7XX_UNKNOWN] = "unknown", 59 [T7XX_READY] = "ready", 60 [T7XX_RESET] = "reset", 61 [T7XX_FASTBOOT_SWITCHING] = "fastboot_switching", 62 [T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download", 63 [T7XX_FASTBOOT_DUMP] = "fastboot_dump", 64 }; 65 66 static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST); 67 68 static ssize_t t7xx_mode_store(struct device *dev, 69 struct device_attribute *attr, 70 const char *buf, size_t count) 71 { 72 struct t7xx_pci_dev *t7xx_dev; 73 struct pci_dev *pdev; 74 enum t7xx_mode mode; 75 int index = 0; 76 77 pdev = to_pci_dev(dev); 78 t7xx_dev = pci_get_drvdata(pdev); 79 if (!t7xx_dev) 80 return -ENODEV; 81 82 mode = READ_ONCE(t7xx_dev->mode); 83 84 index = sysfs_match_string(t7xx_mode_names, buf); 85 if (index == mode) 86 return -EBUSY; 87 88 if (index == T7XX_FASTBOOT_SWITCHING) { 89 if (mode == T7XX_FASTBOOT_DOWNLOAD) 90 return count; 91 92 WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING); 93 pm_runtime_resume(dev); 94 t7xx_reset_device(t7xx_dev, FASTBOOT); 95 } else if (index == T7XX_RESET) { 96 pm_runtime_resume(dev); 97 t7xx_reset_device(t7xx_dev, PLDR); 98 } 99 100 return count; 101 }; 102 103 static ssize_t t7xx_mode_show(struct device *dev, 104 struct device_attribute *attr, 105 char *buf) 106 { 107 enum t7xx_mode mode = T7XX_UNKNOWN; 108 struct t7xx_pci_dev *t7xx_dev; 109 struct pci_dev *pdev; 110 111 pdev = to_pci_dev(dev); 112 t7xx_dev = pci_get_drvdata(pdev); 113 if (!t7xx_dev) 114 return -ENODEV; 115 116 mode = READ_ONCE(t7xx_dev->mode); 117 if (mode < T7XX_MODE_LAST) 118 return sysfs_emit(buf, "%s\n", t7xx_mode_names[mode]); 119 120 return sysfs_emit(buf, "%s\n", t7xx_mode_names[T7XX_UNKNOWN]); 121 } 122 123 static DEVICE_ATTR_RW(t7xx_mode); 124 125 static ssize_t t7xx_debug_ports_store(struct device *dev, 126 struct device_attribute *attr, 127 const char *buf, size_t count) 128 { 129 struct t7xx_pci_dev *t7xx_dev; 130 struct pci_dev *pdev; 131 bool show; 132 int ret; 133 134 pdev = to_pci_dev(dev); 135 t7xx_dev = pci_get_drvdata(pdev); 136 if (!t7xx_dev) 137 return -ENODEV; 138 139 ret = kstrtobool(buf, &show); 140 if (ret < 0) 141 return ret; 142 143 t7xx_proxy_debug_ports_show(t7xx_dev, show); 144 WRITE_ONCE(t7xx_dev->debug_ports_show, show); 145 146 return count; 147 }; 148 149 static ssize_t t7xx_debug_ports_show(struct device *dev, 150 struct device_attribute *attr, 151 char *buf) 152 { 153 struct t7xx_pci_dev *t7xx_dev; 154 struct pci_dev *pdev; 155 bool show; 156 157 pdev = to_pci_dev(dev); 158 t7xx_dev = pci_get_drvdata(pdev); 159 if (!t7xx_dev) 160 return -ENODEV; 161 162 show = READ_ONCE(t7xx_dev->debug_ports_show); 163 164 return sysfs_emit(buf, "%d\n", show); 165 } 166 167 static DEVICE_ATTR_RW(t7xx_debug_ports); 168 169 static struct attribute *t7xx_attr[] = { 170 &dev_attr_t7xx_mode.attr, 171 &dev_attr_t7xx_debug_ports.attr, 172 NULL 173 }; 174 175 static const struct attribute_group t7xx_attribute_group = { 176 .attrs = t7xx_attr, 177 }; 178 179 void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode) 180 { 181 if (!t7xx_dev) 182 return; 183 184 WRITE_ONCE(t7xx_dev->mode, mode); 185 sysfs_notify(&t7xx_dev->pdev->dev.kobj, NULL, "t7xx_mode"); 186 } 187 188 enum t7xx_pm_state { 189 MTK_PM_EXCEPTION, 190 MTK_PM_INIT, /* Device initialized, but handshake not completed */ 191 MTK_PM_SUSPENDED, 192 MTK_PM_RESUMED, 193 }; 194 195 static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable) 196 { 197 void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL; 198 u32 value; 199 200 value = ioread32(ctrl_reg); 201 202 if (enable) 203 value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS; 204 else 205 value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS; 206 207 iowrite32(value, ctrl_reg); 208 } 209 210 static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev) 211 { 212 int ret, val; 213 214 ret = read_poll_timeout(ioread32, val, 215 (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK, 216 PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true, 217 IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); 218 if (ret == -ETIMEDOUT) 219 dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n"); 220 221 return ret; 222 } 223 224 static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev) 225 { 226 struct pci_dev *pdev = t7xx_dev->pdev; 227 228 INIT_LIST_HEAD(&t7xx_dev->md_pm_entities); 229 mutex_init(&t7xx_dev->md_pm_entity_mtx); 230 spin_lock_init(&t7xx_dev->md_pm_lock); 231 init_completion(&t7xx_dev->sleep_lock_acquire); 232 init_completion(&t7xx_dev->pm_sr_ack); 233 init_completion(&t7xx_dev->init_done); 234 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT); 235 236 device_init_wakeup(&pdev->dev, true); 237 dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags | 238 DPM_FLAG_NO_DIRECT_COMPLETE); 239 240 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 241 pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS); 242 pm_runtime_use_autosuspend(&pdev->dev); 243 244 return 0; 245 } 246 247 void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev) 248 { 249 /* Enable the PCIe resource lock only after MD deep sleep is done */ 250 t7xx_mhccif_mask_clr(t7xx_dev, 251 D2H_INT_DS_LOCK_ACK | 252 D2H_INT_SUSPEND_ACK | 253 D2H_INT_RESUME_ACK | 254 D2H_INT_SUSPEND_ACK_AP | 255 D2H_INT_RESUME_ACK_AP); 256 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 257 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 258 259 pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev); 260 pm_runtime_allow(&t7xx_dev->pdev->dev); 261 pm_runtime_put_noidle(&t7xx_dev->pdev->dev); 262 complete_all(&t7xx_dev->init_done); 263 } 264 265 static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev) 266 { 267 /* The device is kept in FSM re-init flow 268 * so just roll back PM setting to the init setting. 269 */ 270 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT); 271 272 pm_runtime_get_noresume(&t7xx_dev->pdev->dev); 273 274 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 275 return t7xx_wait_pm_config(t7xx_dev); 276 } 277 278 void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev) 279 { 280 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 281 t7xx_wait_pm_config(t7xx_dev); 282 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION); 283 } 284 285 int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) 286 { 287 struct md_pm_entity *entity; 288 289 mutex_lock(&t7xx_dev->md_pm_entity_mtx); 290 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 291 if (entity->id == pm_entity->id) { 292 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 293 return -EEXIST; 294 } 295 } 296 297 list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities); 298 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 299 return 0; 300 } 301 302 int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) 303 { 304 struct md_pm_entity *entity, *tmp_entity; 305 306 mutex_lock(&t7xx_dev->md_pm_entity_mtx); 307 list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) { 308 if (entity->id == pm_entity->id) { 309 list_del(&pm_entity->entity); 310 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 311 return 0; 312 } 313 } 314 315 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 316 317 return -ENXIO; 318 } 319 320 int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev) 321 { 322 struct device *dev = &t7xx_dev->pdev->dev; 323 int ret; 324 325 ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire, 326 msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS)); 327 if (!ret) 328 dev_err_ratelimited(dev, "Resource wait complete timed out\n"); 329 330 return ret; 331 } 332 333 /** 334 * t7xx_pci_disable_sleep() - Disable deep sleep capability. 335 * @t7xx_dev: MTK device. 336 * 337 * Lock the deep sleep capability, note that the device can still go into deep sleep 338 * state while device is in D0 state, from the host's point-of-view. 339 * 340 * If device is in deep sleep state, wake up the device and disable deep sleep capability. 341 */ 342 void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev) 343 { 344 unsigned long flags; 345 346 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); 347 t7xx_dev->sleep_disable_count++; 348 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED) 349 goto unlock_and_complete; 350 351 if (t7xx_dev->sleep_disable_count == 1) { 352 u32 status; 353 354 reinit_completion(&t7xx_dev->sleep_lock_acquire); 355 t7xx_dev_set_sleep_capability(t7xx_dev, false); 356 357 status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); 358 if (status & T7XX_PCIE_RESOURCE_STS_MSK) 359 goto unlock_and_complete; 360 361 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK); 362 } 363 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 364 return; 365 366 unlock_and_complete: 367 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 368 complete_all(&t7xx_dev->sleep_lock_acquire); 369 } 370 371 /** 372 * t7xx_pci_enable_sleep() - Enable deep sleep capability. 373 * @t7xx_dev: MTK device. 374 * 375 * After enabling deep sleep, device can enter into deep sleep state. 376 */ 377 void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev) 378 { 379 unsigned long flags; 380 381 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); 382 t7xx_dev->sleep_disable_count--; 383 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED) 384 goto unlock; 385 386 if (t7xx_dev->sleep_disable_count == 0) 387 t7xx_dev_set_sleep_capability(t7xx_dev, true); 388 389 unlock: 390 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 391 } 392 393 static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request) 394 { 395 unsigned long wait_ret; 396 397 reinit_completion(&t7xx_dev->pm_sr_ack); 398 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request); 399 wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack, 400 msecs_to_jiffies(PM_ACK_TIMEOUT_MS)); 401 if (!wait_ret) 402 return -ETIMEDOUT; 403 404 return 0; 405 } 406 407 static int __t7xx_pci_pm_suspend(struct pci_dev *pdev) 408 { 409 enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID; 410 struct t7xx_pci_dev *t7xx_dev; 411 struct md_pm_entity *entity; 412 int ret; 413 414 t7xx_dev = pci_get_drvdata(pdev); 415 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT || 416 READ_ONCE(t7xx_dev->mode) != T7XX_READY) { 417 dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n"); 418 return -EFAULT; 419 } 420 421 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 422 ret = t7xx_wait_pm_config(t7xx_dev); 423 if (ret) { 424 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 425 return ret; 426 } 427 428 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 429 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT); 430 t7xx_dev->rgu_pci_irq_en = false; 431 432 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 433 if (!entity->suspend) 434 continue; 435 436 ret = entity->suspend(t7xx_dev, entity->entity_param); 437 if (ret) { 438 entity_id = entity->id; 439 dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id); 440 goto abort_suspend; 441 } 442 } 443 444 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ); 445 if (ret) { 446 dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret); 447 goto abort_suspend; 448 } 449 450 /* Delay to prevent SAP suspend timeout */ 451 msleep(50); 452 453 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP); 454 if (ret) { 455 t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); 456 dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret); 457 goto abort_suspend; 458 } 459 460 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 461 if (entity->suspend_late) 462 entity->suspend_late(t7xx_dev, entity->entity_param); 463 } 464 465 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 466 return 0; 467 468 abort_suspend: 469 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 470 if (entity_id == entity->id) 471 break; 472 473 if (entity->resume) 474 entity->resume(t7xx_dev, entity->entity_param); 475 } 476 477 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 478 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 479 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 480 return ret; 481 } 482 483 static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev) 484 { 485 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); 486 487 /* Disable interrupt first and let the IPs enable them */ 488 iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0); 489 490 /* Device disables PCIe interrupts during resume and 491 * following function will re-enable PCIe interrupts. 492 */ 493 t7xx_pcie_mac_interrupts_en(t7xx_dev); 494 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT); 495 } 496 497 static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3) 498 { 499 int ret; 500 501 ret = pcim_enable_device(t7xx_dev->pdev); 502 if (ret) 503 return ret; 504 505 t7xx_pcie_mac_atr_init(t7xx_dev); 506 t7xx_pcie_interrupt_reinit(t7xx_dev); 507 508 if (is_d3) { 509 t7xx_mhccif_init(t7xx_dev); 510 t7xx_pci_pm_reinit(t7xx_dev); 511 } 512 513 return 0; 514 } 515 516 static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event) 517 { 518 struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl; 519 struct device *dev = &t7xx_dev->pdev->dev; 520 int ret = -EINVAL; 521 522 switch (event) { 523 case FSM_CMD_STOP: 524 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION); 525 break; 526 527 case FSM_CMD_START: 528 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT); 529 t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT); 530 t7xx_dev->rgu_pci_irq_en = true; 531 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 532 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0); 533 break; 534 535 default: 536 break; 537 } 538 539 if (ret) 540 dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret); 541 542 return ret; 543 } 544 545 int t7xx_pci_reprobe_early(struct t7xx_pci_dev *t7xx_dev) 546 { 547 enum t7xx_mode mode = READ_ONCE(t7xx_dev->mode); 548 int ret; 549 550 if (mode == T7XX_FASTBOOT_DOWNLOAD) 551 pm_runtime_put_noidle(&t7xx_dev->pdev->dev); 552 553 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP); 554 if (ret) 555 return ret; 556 557 return 0; 558 } 559 560 int t7xx_pci_reprobe(struct t7xx_pci_dev *t7xx_dev, bool boot) 561 { 562 int ret; 563 564 ret = t7xx_pcie_reinit(t7xx_dev, boot); 565 if (ret) 566 return ret; 567 568 t7xx_clear_rgu_irq(t7xx_dev); 569 return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START); 570 } 571 572 static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check) 573 { 574 struct t7xx_pci_dev *t7xx_dev; 575 struct md_pm_entity *entity; 576 u32 prev_state; 577 int ret = 0; 578 579 t7xx_dev = pci_get_drvdata(pdev); 580 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) { 581 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 582 return 0; 583 } 584 585 t7xx_pcie_mac_interrupts_en(t7xx_dev); 586 prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE); 587 588 if (state_check) { 589 /* For D3/L3 resume, the device could boot so quickly that the 590 * initial value of the dummy register might be overwritten. 591 * Identify new boots if the ATR source address register is not initialized. 592 */ 593 u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) + 594 ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR); 595 if (prev_state == PM_RESUME_REG_STATE_L3 || 596 (prev_state == PM_RESUME_REG_STATE_INIT && 597 atr_reg_val == ATR_SRC_ADDR_INVALID)) { 598 ret = t7xx_pci_reprobe_early(t7xx_dev); 599 if (ret) 600 return ret; 601 602 return t7xx_pci_reprobe(t7xx_dev, true); 603 } 604 605 if (prev_state == PM_RESUME_REG_STATE_EXP || 606 prev_state == PM_RESUME_REG_STATE_L2_EXP) { 607 if (prev_state == PM_RESUME_REG_STATE_L2_EXP) { 608 ret = t7xx_pcie_reinit(t7xx_dev, false); 609 if (ret) 610 return ret; 611 } 612 613 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 614 t7xx_dev->rgu_pci_irq_en = true; 615 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 616 617 t7xx_mhccif_mask_clr(t7xx_dev, 618 D2H_INT_EXCEPTION_INIT | 619 D2H_INT_EXCEPTION_INIT_DONE | 620 D2H_INT_EXCEPTION_CLEARQ_DONE | 621 D2H_INT_EXCEPTION_ALLQ_RESET | 622 D2H_INT_PORT_ENUM); 623 624 return ret; 625 } 626 627 if (prev_state == PM_RESUME_REG_STATE_L2) { 628 ret = t7xx_pcie_reinit(t7xx_dev, false); 629 if (ret) 630 return ret; 631 632 } else if (prev_state != PM_RESUME_REG_STATE_L1 && 633 prev_state != PM_RESUME_REG_STATE_INIT) { 634 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP); 635 if (ret) 636 return ret; 637 638 t7xx_clear_rgu_irq(t7xx_dev); 639 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 640 return 0; 641 } 642 } 643 644 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 645 t7xx_wait_pm_config(t7xx_dev); 646 647 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 648 if (entity->resume_early) 649 entity->resume_early(t7xx_dev, entity->entity_param); 650 } 651 652 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); 653 if (ret) 654 dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret); 655 656 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP); 657 if (ret) 658 dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret); 659 660 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 661 if (entity->resume) { 662 ret = entity->resume(t7xx_dev, entity->entity_param); 663 if (ret) 664 dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n", 665 entity->id, ret); 666 } 667 } 668 669 t7xx_dev->rgu_pci_irq_en = true; 670 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 671 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 672 pm_runtime_mark_last_busy(&pdev->dev); 673 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 674 675 return ret; 676 } 677 678 static int t7xx_pci_pm_resume_noirq(struct device *dev) 679 { 680 struct pci_dev *pdev = to_pci_dev(dev); 681 struct t7xx_pci_dev *t7xx_dev; 682 683 t7xx_dev = pci_get_drvdata(pdev); 684 t7xx_pcie_mac_interrupts_dis(t7xx_dev); 685 686 return 0; 687 } 688 689 static void t7xx_pci_shutdown(struct pci_dev *pdev) 690 { 691 __t7xx_pci_pm_suspend(pdev); 692 } 693 694 static int t7xx_pci_pm_prepare(struct device *dev) 695 { 696 struct pci_dev *pdev = to_pci_dev(dev); 697 struct t7xx_pci_dev *t7xx_dev; 698 699 t7xx_dev = pci_get_drvdata(pdev); 700 if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) { 701 dev_warn(dev, "Not ready for system sleep.\n"); 702 return -ETIMEDOUT; 703 } 704 705 return 0; 706 } 707 708 static int t7xx_pci_pm_suspend(struct device *dev) 709 { 710 return __t7xx_pci_pm_suspend(to_pci_dev(dev)); 711 } 712 713 static int t7xx_pci_pm_resume(struct device *dev) 714 { 715 return __t7xx_pci_pm_resume(to_pci_dev(dev), true); 716 } 717 718 static int t7xx_pci_pm_thaw(struct device *dev) 719 { 720 return __t7xx_pci_pm_resume(to_pci_dev(dev), false); 721 } 722 723 static int t7xx_pci_pm_runtime_suspend(struct device *dev) 724 { 725 return __t7xx_pci_pm_suspend(to_pci_dev(dev)); 726 } 727 728 static int t7xx_pci_pm_runtime_resume(struct device *dev) 729 { 730 return __t7xx_pci_pm_resume(to_pci_dev(dev), true); 731 } 732 733 static const struct dev_pm_ops t7xx_pci_pm_ops = { 734 .prepare = t7xx_pci_pm_prepare, 735 .suspend = t7xx_pci_pm_suspend, 736 .resume = t7xx_pci_pm_resume, 737 .resume_noirq = t7xx_pci_pm_resume_noirq, 738 .freeze = t7xx_pci_pm_suspend, 739 .thaw = t7xx_pci_pm_thaw, 740 .poweroff = t7xx_pci_pm_suspend, 741 .restore = t7xx_pci_pm_resume, 742 .restore_noirq = t7xx_pci_pm_resume_noirq, 743 .runtime_suspend = t7xx_pci_pm_runtime_suspend, 744 .runtime_resume = t7xx_pci_pm_runtime_resume 745 }; 746 747 static int t7xx_request_irq(struct pci_dev *pdev) 748 { 749 struct t7xx_pci_dev *t7xx_dev; 750 int ret = 0, i; 751 752 t7xx_dev = pci_get_drvdata(pdev); 753 754 for (i = 0; i < EXT_INT_NUM; i++) { 755 const char *irq_descr; 756 int irq_vec; 757 758 if (!t7xx_dev->intr_handler[i]) 759 continue; 760 761 irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d", 762 dev_driver_string(&pdev->dev), i); 763 if (!irq_descr) { 764 ret = -ENOMEM; 765 break; 766 } 767 768 irq_vec = pci_irq_vector(pdev, i); 769 ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i], 770 t7xx_dev->intr_thread[i], 0, irq_descr, 771 t7xx_dev->callback_param[i]); 772 if (ret) { 773 dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret); 774 break; 775 } 776 } 777 778 if (ret) { 779 while (i--) { 780 if (!t7xx_dev->intr_handler[i]) 781 continue; 782 783 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]); 784 } 785 } 786 787 return ret; 788 } 789 790 static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev) 791 { 792 struct pci_dev *pdev = t7xx_dev->pdev; 793 int ret; 794 795 /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */ 796 ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX); 797 if (ret < 0) { 798 dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret); 799 return ret; 800 } 801 802 ret = t7xx_request_irq(pdev); 803 if (ret) { 804 pci_free_irq_vectors(pdev); 805 return ret; 806 } 807 808 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); 809 return 0; 810 } 811 812 static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev) 813 { 814 int ret, i; 815 816 if (!t7xx_dev->pdev->msix_cap) 817 return -EINVAL; 818 819 ret = t7xx_setup_msix(t7xx_dev); 820 if (ret) 821 return ret; 822 823 /* IPs enable interrupts when ready */ 824 for (i = 0; i < EXT_INT_NUM; i++) 825 t7xx_pcie_mac_set_int(t7xx_dev, i); 826 827 return 0; 828 } 829 830 static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev) 831 { 832 t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base + 833 INFRACFG_AO_DEV_CHIP - 834 t7xx_dev->base_addr.pcie_dev_reg_trsl_addr; 835 } 836 837 static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 838 { 839 struct t7xx_pci_dev *t7xx_dev; 840 void __iomem *iomem; 841 int ret; 842 843 t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL); 844 if (!t7xx_dev) 845 return -ENOMEM; 846 847 pci_set_drvdata(pdev, t7xx_dev); 848 t7xx_dev->pdev = pdev; 849 850 ret = pcim_enable_device(pdev); 851 if (ret) 852 return ret; 853 854 pci_set_master(pdev); 855 856 iomem = pcim_iomap_region(pdev, T7XX_PCI_IREG_BASE, DRIVER_NAME); 857 ret = PTR_ERR_OR_ZERO(iomem); 858 if (ret) { 859 dev_err(&pdev->dev, "Could not request IREG BAR: %d\n", ret); 860 return -ENOMEM; 861 } 862 IREG_BASE(t7xx_dev) = iomem; 863 864 iomem = pcim_iomap_region(pdev, T7XX_PCI_EREG_BASE, DRIVER_NAME); 865 ret = PTR_ERR_OR_ZERO(iomem); 866 if (ret) { 867 dev_err(&pdev->dev, "Could not request EREG BAR: %d\n", ret); 868 return -ENOMEM; 869 } 870 t7xx_dev->base_addr.pcie_ext_reg_base = iomem; 871 872 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 873 if (ret) { 874 dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret); 875 return ret; 876 } 877 878 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 879 if (ret) { 880 dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret); 881 return ret; 882 } 883 884 ret = t7xx_pci_pm_init(t7xx_dev); 885 if (ret) 886 return ret; 887 888 t7xx_pcie_mac_atr_init(t7xx_dev); 889 t7xx_pci_infracfg_ao_calc(t7xx_dev); 890 t7xx_mhccif_init(t7xx_dev); 891 892 ret = t7xx_md_init(t7xx_dev); 893 if (ret) 894 return ret; 895 896 t7xx_pcie_mac_interrupts_dis(t7xx_dev); 897 898 ret = sysfs_create_group(&t7xx_dev->pdev->dev.kobj, 899 &t7xx_attribute_group); 900 if (ret) 901 goto err_md_exit; 902 903 ret = t7xx_interrupt_init(t7xx_dev); 904 if (ret) 905 goto err_remove_group; 906 907 908 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT); 909 t7xx_pcie_mac_interrupts_en(t7xx_dev); 910 911 return 0; 912 913 err_remove_group: 914 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj, 915 &t7xx_attribute_group); 916 917 err_md_exit: 918 t7xx_md_exit(t7xx_dev); 919 return ret; 920 } 921 922 static void t7xx_pci_remove(struct pci_dev *pdev) 923 { 924 struct t7xx_pci_dev *t7xx_dev; 925 int i; 926 927 t7xx_dev = pci_get_drvdata(pdev); 928 929 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj, 930 &t7xx_attribute_group); 931 t7xx_md_exit(t7xx_dev); 932 933 for (i = 0; i < EXT_INT_NUM; i++) { 934 if (!t7xx_dev->intr_handler[i]) 935 continue; 936 937 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]); 938 } 939 940 pci_free_irq_vectors(t7xx_dev->pdev); 941 } 942 943 static const struct pci_device_id t7xx_pci_table[] = { 944 { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) }, 945 { PCI_DEVICE(0x03f0, 0x09c8) }, // HP DRMR-H01 946 { PCI_DEVICE(0x14c0, 0x4d75) }, // Dell DW5933e 947 { } 948 }; 949 MODULE_DEVICE_TABLE(pci, t7xx_pci_table); 950 951 static struct pci_driver t7xx_pci_driver = { 952 .name = DRIVER_NAME, 953 .id_table = t7xx_pci_table, 954 .probe = t7xx_pci_probe, 955 .remove = t7xx_pci_remove, 956 .driver.pm = &t7xx_pci_pm_ops, 957 .shutdown = t7xx_pci_shutdown, 958 }; 959 960 module_pci_driver(t7xx_pci_driver); 961 962 MODULE_AUTHOR("MediaTek Inc"); 963 MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver"); 964 MODULE_LICENSE("GPL"); 965