1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021, MediaTek Inc. 4 * Copyright (c) 2021-2022, Intel Corporation. 5 * 6 * Authors: 7 * Haijun Liu <haijun.liu@mediatek.com> 8 * Ricardo Martinez <ricardo.martinez@linux.intel.com> 9 * Sreehari Kancharla <sreehari.kancharla@intel.com> 10 * 11 * Contributors: 12 * Amir Hanania <amir.hanania@intel.com> 13 * Andy Shevchenko <andriy.shevchenko@linux.intel.com> 14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15 * Eliot Lee <eliot.lee@intel.com> 16 * Moises Veleta <moises.veleta@intel.com> 17 */ 18 19 #include <linux/atomic.h> 20 #include <linux/bits.h> 21 #include <linux/completion.h> 22 #include <linux/device.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/gfp.h> 25 #include <linux/interrupt.h> 26 #include <linux/io.h> 27 #include <linux/iopoll.h> 28 #include <linux/jiffies.h> 29 #include <linux/list.h> 30 #include <linux/module.h> 31 #include <linux/mutex.h> 32 #include <linux/pci.h> 33 #include <linux/pm.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/pm_wakeup.h> 36 #include <linux/spinlock.h> 37 38 #include "t7xx_mhccif.h" 39 #include "t7xx_modem_ops.h" 40 #include "t7xx_pci.h" 41 #include "t7xx_pcie_mac.h" 42 #include "t7xx_reg.h" 43 #include "t7xx_state_monitor.h" 44 45 #define T7XX_PCI_IREG_BASE 0 46 #define T7XX_PCI_EREG_BASE 2 47 48 #define T7XX_INIT_TIMEOUT 20 49 #define PM_SLEEP_DIS_TIMEOUT_MS 20 50 #define PM_ACK_TIMEOUT_MS 1500 51 #define PM_AUTOSUSPEND_MS 20000 52 #define PM_RESOURCE_POLL_TIMEOUT_US 10000 53 #define PM_RESOURCE_POLL_STEP_US 100 54 55 static const char * const t7xx_mode_names[] = { 56 [T7XX_UNKNOWN] = "unknown", 57 [T7XX_READY] = "ready", 58 [T7XX_RESET] = "reset", 59 [T7XX_FASTBOOT_SWITCHING] = "fastboot_switching", 60 [T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download", 61 [T7XX_FASTBOOT_DUMP] = "fastboot_dump", 62 }; 63 64 static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST); 65 66 static ssize_t t7xx_mode_store(struct device *dev, 67 struct device_attribute *attr, 68 const char *buf, size_t count) 69 { 70 struct t7xx_pci_dev *t7xx_dev; 71 struct pci_dev *pdev; 72 int index = 0; 73 74 pdev = to_pci_dev(dev); 75 t7xx_dev = pci_get_drvdata(pdev); 76 if (!t7xx_dev) 77 return -ENODEV; 78 79 index = sysfs_match_string(t7xx_mode_names, buf); 80 if (index == T7XX_FASTBOOT_SWITCHING) { 81 WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING); 82 } else if (index == T7XX_RESET) { 83 WRITE_ONCE(t7xx_dev->mode, T7XX_RESET); 84 t7xx_acpi_pldr_func(t7xx_dev); 85 } 86 87 return count; 88 }; 89 90 static ssize_t t7xx_mode_show(struct device *dev, 91 struct device_attribute *attr, 92 char *buf) 93 { 94 enum t7xx_mode mode = T7XX_UNKNOWN; 95 struct t7xx_pci_dev *t7xx_dev; 96 struct pci_dev *pdev; 97 98 pdev = to_pci_dev(dev); 99 t7xx_dev = pci_get_drvdata(pdev); 100 if (!t7xx_dev) 101 return -ENODEV; 102 103 mode = READ_ONCE(t7xx_dev->mode); 104 if (mode < T7XX_MODE_LAST) 105 return sysfs_emit(buf, "%s\n", t7xx_mode_names[mode]); 106 107 return sysfs_emit(buf, "%s\n", t7xx_mode_names[T7XX_UNKNOWN]); 108 } 109 110 static DEVICE_ATTR_RW(t7xx_mode); 111 112 static struct attribute *t7xx_mode_attr[] = { 113 &dev_attr_t7xx_mode.attr, 114 NULL 115 }; 116 117 static const struct attribute_group t7xx_mode_attribute_group = { 118 .attrs = t7xx_mode_attr, 119 }; 120 121 void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode) 122 { 123 if (!t7xx_dev) 124 return; 125 126 WRITE_ONCE(t7xx_dev->mode, mode); 127 sysfs_notify(&t7xx_dev->pdev->dev.kobj, NULL, "t7xx_mode"); 128 } 129 130 enum t7xx_pm_state { 131 MTK_PM_EXCEPTION, 132 MTK_PM_INIT, /* Device initialized, but handshake not completed */ 133 MTK_PM_SUSPENDED, 134 MTK_PM_RESUMED, 135 }; 136 137 static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable) 138 { 139 void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL; 140 u32 value; 141 142 value = ioread32(ctrl_reg); 143 144 if (enable) 145 value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS; 146 else 147 value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS; 148 149 iowrite32(value, ctrl_reg); 150 } 151 152 static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev) 153 { 154 int ret, val; 155 156 ret = read_poll_timeout(ioread32, val, 157 (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK, 158 PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true, 159 IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); 160 if (ret == -ETIMEDOUT) 161 dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n"); 162 163 return ret; 164 } 165 166 static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev) 167 { 168 struct pci_dev *pdev = t7xx_dev->pdev; 169 170 INIT_LIST_HEAD(&t7xx_dev->md_pm_entities); 171 mutex_init(&t7xx_dev->md_pm_entity_mtx); 172 spin_lock_init(&t7xx_dev->md_pm_lock); 173 init_completion(&t7xx_dev->sleep_lock_acquire); 174 init_completion(&t7xx_dev->pm_sr_ack); 175 init_completion(&t7xx_dev->init_done); 176 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT); 177 178 device_init_wakeup(&pdev->dev, true); 179 dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags | 180 DPM_FLAG_NO_DIRECT_COMPLETE); 181 182 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 183 pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS); 184 pm_runtime_use_autosuspend(&pdev->dev); 185 186 return 0; 187 } 188 189 void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev) 190 { 191 /* Enable the PCIe resource lock only after MD deep sleep is done */ 192 t7xx_mhccif_mask_clr(t7xx_dev, 193 D2H_INT_DS_LOCK_ACK | 194 D2H_INT_SUSPEND_ACK | 195 D2H_INT_RESUME_ACK | 196 D2H_INT_SUSPEND_ACK_AP | 197 D2H_INT_RESUME_ACK_AP); 198 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 199 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 200 201 pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev); 202 pm_runtime_allow(&t7xx_dev->pdev->dev); 203 pm_runtime_put_noidle(&t7xx_dev->pdev->dev); 204 complete_all(&t7xx_dev->init_done); 205 } 206 207 static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev) 208 { 209 /* The device is kept in FSM re-init flow 210 * so just roll back PM setting to the init setting. 211 */ 212 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT); 213 214 pm_runtime_get_noresume(&t7xx_dev->pdev->dev); 215 216 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 217 return t7xx_wait_pm_config(t7xx_dev); 218 } 219 220 void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev) 221 { 222 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 223 t7xx_wait_pm_config(t7xx_dev); 224 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION); 225 } 226 227 int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) 228 { 229 struct md_pm_entity *entity; 230 231 mutex_lock(&t7xx_dev->md_pm_entity_mtx); 232 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 233 if (entity->id == pm_entity->id) { 234 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 235 return -EEXIST; 236 } 237 } 238 239 list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities); 240 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 241 return 0; 242 } 243 244 int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) 245 { 246 struct md_pm_entity *entity, *tmp_entity; 247 248 mutex_lock(&t7xx_dev->md_pm_entity_mtx); 249 list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) { 250 if (entity->id == pm_entity->id) { 251 list_del(&pm_entity->entity); 252 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 253 return 0; 254 } 255 } 256 257 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 258 259 return -ENXIO; 260 } 261 262 int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev) 263 { 264 struct device *dev = &t7xx_dev->pdev->dev; 265 int ret; 266 267 ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire, 268 msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS)); 269 if (!ret) 270 dev_err_ratelimited(dev, "Resource wait complete timed out\n"); 271 272 return ret; 273 } 274 275 /** 276 * t7xx_pci_disable_sleep() - Disable deep sleep capability. 277 * @t7xx_dev: MTK device. 278 * 279 * Lock the deep sleep capability, note that the device can still go into deep sleep 280 * state while device is in D0 state, from the host's point-of-view. 281 * 282 * If device is in deep sleep state, wake up the device and disable deep sleep capability. 283 */ 284 void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev) 285 { 286 unsigned long flags; 287 288 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); 289 t7xx_dev->sleep_disable_count++; 290 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED) 291 goto unlock_and_complete; 292 293 if (t7xx_dev->sleep_disable_count == 1) { 294 u32 status; 295 296 reinit_completion(&t7xx_dev->sleep_lock_acquire); 297 t7xx_dev_set_sleep_capability(t7xx_dev, false); 298 299 status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); 300 if (status & T7XX_PCIE_RESOURCE_STS_MSK) 301 goto unlock_and_complete; 302 303 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK); 304 } 305 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 306 return; 307 308 unlock_and_complete: 309 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 310 complete_all(&t7xx_dev->sleep_lock_acquire); 311 } 312 313 /** 314 * t7xx_pci_enable_sleep() - Enable deep sleep capability. 315 * @t7xx_dev: MTK device. 316 * 317 * After enabling deep sleep, device can enter into deep sleep state. 318 */ 319 void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev) 320 { 321 unsigned long flags; 322 323 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); 324 t7xx_dev->sleep_disable_count--; 325 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED) 326 goto unlock; 327 328 if (t7xx_dev->sleep_disable_count == 0) 329 t7xx_dev_set_sleep_capability(t7xx_dev, true); 330 331 unlock: 332 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 333 } 334 335 static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request) 336 { 337 unsigned long wait_ret; 338 339 reinit_completion(&t7xx_dev->pm_sr_ack); 340 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request); 341 wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack, 342 msecs_to_jiffies(PM_ACK_TIMEOUT_MS)); 343 if (!wait_ret) 344 return -ETIMEDOUT; 345 346 return 0; 347 } 348 349 static int __t7xx_pci_pm_suspend(struct pci_dev *pdev) 350 { 351 enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID; 352 struct t7xx_pci_dev *t7xx_dev; 353 struct md_pm_entity *entity; 354 int ret; 355 356 t7xx_dev = pci_get_drvdata(pdev); 357 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT || 358 READ_ONCE(t7xx_dev->mode) != T7XX_READY) { 359 dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n"); 360 return -EFAULT; 361 } 362 363 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 364 ret = t7xx_wait_pm_config(t7xx_dev); 365 if (ret) { 366 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 367 return ret; 368 } 369 370 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 371 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT); 372 t7xx_dev->rgu_pci_irq_en = false; 373 374 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 375 if (!entity->suspend) 376 continue; 377 378 ret = entity->suspend(t7xx_dev, entity->entity_param); 379 if (ret) { 380 entity_id = entity->id; 381 dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id); 382 goto abort_suspend; 383 } 384 } 385 386 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ); 387 if (ret) { 388 dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret); 389 goto abort_suspend; 390 } 391 392 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP); 393 if (ret) { 394 t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); 395 dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret); 396 goto abort_suspend; 397 } 398 399 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 400 if (entity->suspend_late) 401 entity->suspend_late(t7xx_dev, entity->entity_param); 402 } 403 404 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 405 return 0; 406 407 abort_suspend: 408 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 409 if (entity_id == entity->id) 410 break; 411 412 if (entity->resume) 413 entity->resume(t7xx_dev, entity->entity_param); 414 } 415 416 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 417 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 418 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 419 return ret; 420 } 421 422 static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev) 423 { 424 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); 425 426 /* Disable interrupt first and let the IPs enable them */ 427 iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0); 428 429 /* Device disables PCIe interrupts during resume and 430 * following function will re-enable PCIe interrupts. 431 */ 432 t7xx_pcie_mac_interrupts_en(t7xx_dev); 433 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT); 434 } 435 436 static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3) 437 { 438 int ret; 439 440 ret = pcim_enable_device(t7xx_dev->pdev); 441 if (ret) 442 return ret; 443 444 t7xx_pcie_mac_atr_init(t7xx_dev); 445 t7xx_pcie_interrupt_reinit(t7xx_dev); 446 447 if (is_d3) { 448 t7xx_mhccif_init(t7xx_dev); 449 return t7xx_pci_pm_reinit(t7xx_dev); 450 } 451 452 return 0; 453 } 454 455 static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event) 456 { 457 struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl; 458 struct device *dev = &t7xx_dev->pdev->dev; 459 int ret = -EINVAL; 460 461 switch (event) { 462 case FSM_CMD_STOP: 463 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION); 464 break; 465 466 case FSM_CMD_START: 467 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT); 468 t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT); 469 t7xx_dev->rgu_pci_irq_en = true; 470 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 471 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0); 472 break; 473 474 default: 475 break; 476 } 477 478 if (ret) 479 dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret); 480 481 return ret; 482 } 483 484 static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check) 485 { 486 struct t7xx_pci_dev *t7xx_dev; 487 struct md_pm_entity *entity; 488 u32 prev_state; 489 int ret = 0; 490 491 t7xx_dev = pci_get_drvdata(pdev); 492 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) { 493 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 494 return 0; 495 } 496 497 t7xx_pcie_mac_interrupts_en(t7xx_dev); 498 prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE); 499 500 if (state_check) { 501 /* For D3/L3 resume, the device could boot so quickly that the 502 * initial value of the dummy register might be overwritten. 503 * Identify new boots if the ATR source address register is not initialized. 504 */ 505 u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) + 506 ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR); 507 if (prev_state == PM_RESUME_REG_STATE_L3 || 508 (prev_state == PM_RESUME_REG_STATE_INIT && 509 atr_reg_val == ATR_SRC_ADDR_INVALID)) { 510 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP); 511 if (ret) 512 return ret; 513 514 ret = t7xx_pcie_reinit(t7xx_dev, true); 515 if (ret) 516 return ret; 517 518 t7xx_clear_rgu_irq(t7xx_dev); 519 return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START); 520 } 521 522 if (prev_state == PM_RESUME_REG_STATE_EXP || 523 prev_state == PM_RESUME_REG_STATE_L2_EXP) { 524 if (prev_state == PM_RESUME_REG_STATE_L2_EXP) { 525 ret = t7xx_pcie_reinit(t7xx_dev, false); 526 if (ret) 527 return ret; 528 } 529 530 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 531 t7xx_dev->rgu_pci_irq_en = true; 532 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 533 534 t7xx_mhccif_mask_clr(t7xx_dev, 535 D2H_INT_EXCEPTION_INIT | 536 D2H_INT_EXCEPTION_INIT_DONE | 537 D2H_INT_EXCEPTION_CLEARQ_DONE | 538 D2H_INT_EXCEPTION_ALLQ_RESET | 539 D2H_INT_PORT_ENUM); 540 541 return ret; 542 } 543 544 if (prev_state == PM_RESUME_REG_STATE_L2) { 545 ret = t7xx_pcie_reinit(t7xx_dev, false); 546 if (ret) 547 return ret; 548 549 } else if (prev_state != PM_RESUME_REG_STATE_L1 && 550 prev_state != PM_RESUME_REG_STATE_INIT) { 551 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP); 552 if (ret) 553 return ret; 554 555 t7xx_clear_rgu_irq(t7xx_dev); 556 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 557 return 0; 558 } 559 } 560 561 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 562 t7xx_wait_pm_config(t7xx_dev); 563 564 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 565 if (entity->resume_early) 566 entity->resume_early(t7xx_dev, entity->entity_param); 567 } 568 569 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); 570 if (ret) 571 dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret); 572 573 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP); 574 if (ret) 575 dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret); 576 577 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 578 if (entity->resume) { 579 ret = entity->resume(t7xx_dev, entity->entity_param); 580 if (ret) 581 dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n", 582 entity->id, ret); 583 } 584 } 585 586 t7xx_dev->rgu_pci_irq_en = true; 587 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 588 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 589 pm_runtime_mark_last_busy(&pdev->dev); 590 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 591 592 return ret; 593 } 594 595 static int t7xx_pci_pm_resume_noirq(struct device *dev) 596 { 597 struct pci_dev *pdev = to_pci_dev(dev); 598 struct t7xx_pci_dev *t7xx_dev; 599 600 t7xx_dev = pci_get_drvdata(pdev); 601 t7xx_pcie_mac_interrupts_dis(t7xx_dev); 602 603 return 0; 604 } 605 606 static void t7xx_pci_shutdown(struct pci_dev *pdev) 607 { 608 __t7xx_pci_pm_suspend(pdev); 609 } 610 611 static int t7xx_pci_pm_prepare(struct device *dev) 612 { 613 struct pci_dev *pdev = to_pci_dev(dev); 614 struct t7xx_pci_dev *t7xx_dev; 615 616 t7xx_dev = pci_get_drvdata(pdev); 617 if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) { 618 dev_warn(dev, "Not ready for system sleep.\n"); 619 return -ETIMEDOUT; 620 } 621 622 return 0; 623 } 624 625 static int t7xx_pci_pm_suspend(struct device *dev) 626 { 627 return __t7xx_pci_pm_suspend(to_pci_dev(dev)); 628 } 629 630 static int t7xx_pci_pm_resume(struct device *dev) 631 { 632 return __t7xx_pci_pm_resume(to_pci_dev(dev), true); 633 } 634 635 static int t7xx_pci_pm_thaw(struct device *dev) 636 { 637 return __t7xx_pci_pm_resume(to_pci_dev(dev), false); 638 } 639 640 static int t7xx_pci_pm_runtime_suspend(struct device *dev) 641 { 642 return __t7xx_pci_pm_suspend(to_pci_dev(dev)); 643 } 644 645 static int t7xx_pci_pm_runtime_resume(struct device *dev) 646 { 647 return __t7xx_pci_pm_resume(to_pci_dev(dev), true); 648 } 649 650 static const struct dev_pm_ops t7xx_pci_pm_ops = { 651 .prepare = t7xx_pci_pm_prepare, 652 .suspend = t7xx_pci_pm_suspend, 653 .resume = t7xx_pci_pm_resume, 654 .resume_noirq = t7xx_pci_pm_resume_noirq, 655 .freeze = t7xx_pci_pm_suspend, 656 .thaw = t7xx_pci_pm_thaw, 657 .poweroff = t7xx_pci_pm_suspend, 658 .restore = t7xx_pci_pm_resume, 659 .restore_noirq = t7xx_pci_pm_resume_noirq, 660 .runtime_suspend = t7xx_pci_pm_runtime_suspend, 661 .runtime_resume = t7xx_pci_pm_runtime_resume 662 }; 663 664 static int t7xx_request_irq(struct pci_dev *pdev) 665 { 666 struct t7xx_pci_dev *t7xx_dev; 667 int ret = 0, i; 668 669 t7xx_dev = pci_get_drvdata(pdev); 670 671 for (i = 0; i < EXT_INT_NUM; i++) { 672 const char *irq_descr; 673 int irq_vec; 674 675 if (!t7xx_dev->intr_handler[i]) 676 continue; 677 678 irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d", 679 dev_driver_string(&pdev->dev), i); 680 if (!irq_descr) { 681 ret = -ENOMEM; 682 break; 683 } 684 685 irq_vec = pci_irq_vector(pdev, i); 686 ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i], 687 t7xx_dev->intr_thread[i], 0, irq_descr, 688 t7xx_dev->callback_param[i]); 689 if (ret) { 690 dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret); 691 break; 692 } 693 } 694 695 if (ret) { 696 while (i--) { 697 if (!t7xx_dev->intr_handler[i]) 698 continue; 699 700 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]); 701 } 702 } 703 704 return ret; 705 } 706 707 static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev) 708 { 709 struct pci_dev *pdev = t7xx_dev->pdev; 710 int ret; 711 712 /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */ 713 ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX); 714 if (ret < 0) { 715 dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret); 716 return ret; 717 } 718 719 ret = t7xx_request_irq(pdev); 720 if (ret) { 721 pci_free_irq_vectors(pdev); 722 return ret; 723 } 724 725 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); 726 return 0; 727 } 728 729 static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev) 730 { 731 int ret, i; 732 733 if (!t7xx_dev->pdev->msix_cap) 734 return -EINVAL; 735 736 ret = t7xx_setup_msix(t7xx_dev); 737 if (ret) 738 return ret; 739 740 /* IPs enable interrupts when ready */ 741 for (i = 0; i < EXT_INT_NUM; i++) 742 t7xx_pcie_mac_set_int(t7xx_dev, i); 743 744 return 0; 745 } 746 747 static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev) 748 { 749 t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base + 750 INFRACFG_AO_DEV_CHIP - 751 t7xx_dev->base_addr.pcie_dev_reg_trsl_addr; 752 } 753 754 static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 755 { 756 struct t7xx_pci_dev *t7xx_dev; 757 int ret; 758 759 t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL); 760 if (!t7xx_dev) 761 return -ENOMEM; 762 763 pci_set_drvdata(pdev, t7xx_dev); 764 t7xx_dev->pdev = pdev; 765 766 ret = pcim_enable_device(pdev); 767 if (ret) 768 return ret; 769 770 pci_set_master(pdev); 771 772 ret = pcim_iomap_regions(pdev, BIT(T7XX_PCI_IREG_BASE) | BIT(T7XX_PCI_EREG_BASE), 773 pci_name(pdev)); 774 if (ret) { 775 dev_err(&pdev->dev, "Could not request BARs: %d\n", ret); 776 return -ENOMEM; 777 } 778 779 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 780 if (ret) { 781 dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret); 782 return ret; 783 } 784 785 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 786 if (ret) { 787 dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret); 788 return ret; 789 } 790 791 IREG_BASE(t7xx_dev) = pcim_iomap_table(pdev)[T7XX_PCI_IREG_BASE]; 792 t7xx_dev->base_addr.pcie_ext_reg_base = pcim_iomap_table(pdev)[T7XX_PCI_EREG_BASE]; 793 794 ret = t7xx_pci_pm_init(t7xx_dev); 795 if (ret) 796 return ret; 797 798 t7xx_pcie_mac_atr_init(t7xx_dev); 799 t7xx_pci_infracfg_ao_calc(t7xx_dev); 800 t7xx_mhccif_init(t7xx_dev); 801 802 ret = t7xx_md_init(t7xx_dev); 803 if (ret) 804 return ret; 805 806 t7xx_pcie_mac_interrupts_dis(t7xx_dev); 807 808 ret = sysfs_create_group(&t7xx_dev->pdev->dev.kobj, 809 &t7xx_mode_attribute_group); 810 if (ret) 811 goto err_md_exit; 812 813 ret = t7xx_interrupt_init(t7xx_dev); 814 if (ret) 815 goto err_remove_group; 816 817 818 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT); 819 t7xx_pcie_mac_interrupts_en(t7xx_dev); 820 821 return 0; 822 823 err_remove_group: 824 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj, 825 &t7xx_mode_attribute_group); 826 827 err_md_exit: 828 t7xx_md_exit(t7xx_dev); 829 return ret; 830 } 831 832 static void t7xx_pci_remove(struct pci_dev *pdev) 833 { 834 struct t7xx_pci_dev *t7xx_dev; 835 int i; 836 837 t7xx_dev = pci_get_drvdata(pdev); 838 839 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj, 840 &t7xx_mode_attribute_group); 841 t7xx_md_exit(t7xx_dev); 842 843 for (i = 0; i < EXT_INT_NUM; i++) { 844 if (!t7xx_dev->intr_handler[i]) 845 continue; 846 847 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]); 848 } 849 850 pci_free_irq_vectors(t7xx_dev->pdev); 851 } 852 853 static const struct pci_device_id t7xx_pci_table[] = { 854 { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) }, 855 { } 856 }; 857 MODULE_DEVICE_TABLE(pci, t7xx_pci_table); 858 859 static struct pci_driver t7xx_pci_driver = { 860 .name = "mtk_t7xx", 861 .id_table = t7xx_pci_table, 862 .probe = t7xx_pci_probe, 863 .remove = t7xx_pci_remove, 864 .driver.pm = &t7xx_pci_pm_ops, 865 .shutdown = t7xx_pci_shutdown, 866 }; 867 868 module_pci_driver(t7xx_pci_driver); 869 870 MODULE_AUTHOR("MediaTek Inc"); 871 MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver"); 872 MODULE_LICENSE("GPL"); 873