1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021, MediaTek Inc. 4 * Copyright (c) 2021-2022, Intel Corporation. 5 * 6 * Authors: 7 * Haijun Liu <haijun.liu@mediatek.com> 8 * Ricardo Martinez <ricardo.martinez@linux.intel.com> 9 * Sreehari Kancharla <sreehari.kancharla@intel.com> 10 * 11 * Contributors: 12 * Amir Hanania <amir.hanania@intel.com> 13 * Andy Shevchenko <andriy.shevchenko@linux.intel.com> 14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15 * Eliot Lee <eliot.lee@intel.com> 16 * Moises Veleta <moises.veleta@intel.com> 17 */ 18 19 #include <linux/atomic.h> 20 #include <linux/bits.h> 21 #include <linux/completion.h> 22 #include <linux/device.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/gfp.h> 25 #include <linux/interrupt.h> 26 #include <linux/io.h> 27 #include <linux/iopoll.h> 28 #include <linux/jiffies.h> 29 #include <linux/list.h> 30 #include <linux/module.h> 31 #include <linux/mutex.h> 32 #include <linux/pci.h> 33 #include <linux/pm.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/pm_wakeup.h> 36 #include <linux/spinlock.h> 37 38 #include "t7xx_mhccif.h" 39 #include "t7xx_modem_ops.h" 40 #include "t7xx_pci.h" 41 #include "t7xx_pcie_mac.h" 42 #include "t7xx_reg.h" 43 #include "t7xx_state_monitor.h" 44 #include "t7xx_port_proxy.h" 45 46 #define DRIVER_NAME "mtk_t7xx" 47 48 #define T7XX_PCI_IREG_BASE 0 49 #define T7XX_PCI_EREG_BASE 2 50 51 #define T7XX_INIT_TIMEOUT 20 52 #define PM_SLEEP_DIS_TIMEOUT_MS 20 53 #define PM_ACK_TIMEOUT_MS 1500 54 #define PM_AUTOSUSPEND_MS 5000 55 #define PM_RESOURCE_POLL_TIMEOUT_US 10000 56 #define PM_RESOURCE_POLL_STEP_US 100 57 58 static const char * const t7xx_mode_names[] = { 59 [T7XX_UNKNOWN] = "unknown", 60 [T7XX_READY] = "ready", 61 [T7XX_RESET] = "reset", 62 [T7XX_FASTBOOT_SWITCHING] = "fastboot_switching", 63 [T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download", 64 [T7XX_FASTBOOT_DUMP] = "fastboot_dump", 65 }; 66 67 static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST); 68 69 static ssize_t t7xx_mode_store(struct device *dev, 70 struct device_attribute *attr, 71 const char *buf, size_t count) 72 { 73 struct t7xx_pci_dev *t7xx_dev; 74 struct pci_dev *pdev; 75 enum t7xx_mode mode; 76 int index = 0; 77 78 pdev = to_pci_dev(dev); 79 t7xx_dev = pci_get_drvdata(pdev); 80 if (!t7xx_dev) 81 return -ENODEV; 82 83 mode = READ_ONCE(t7xx_dev->mode); 84 85 index = sysfs_match_string(t7xx_mode_names, buf); 86 if (index == mode) 87 return -EBUSY; 88 89 if (index == T7XX_FASTBOOT_SWITCHING) { 90 if (mode == T7XX_FASTBOOT_DOWNLOAD) 91 return count; 92 93 WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING); 94 pm_runtime_resume(dev); 95 t7xx_reset_device(t7xx_dev, FASTBOOT); 96 } else if (index == T7XX_RESET) { 97 pm_runtime_resume(dev); 98 t7xx_reset_device(t7xx_dev, PLDR); 99 } 100 101 return count; 102 }; 103 104 static ssize_t t7xx_mode_show(struct device *dev, 105 struct device_attribute *attr, 106 char *buf) 107 { 108 enum t7xx_mode mode = T7XX_UNKNOWN; 109 struct t7xx_pci_dev *t7xx_dev; 110 struct pci_dev *pdev; 111 112 pdev = to_pci_dev(dev); 113 t7xx_dev = pci_get_drvdata(pdev); 114 if (!t7xx_dev) 115 return -ENODEV; 116 117 mode = READ_ONCE(t7xx_dev->mode); 118 if (mode < T7XX_MODE_LAST) 119 return sysfs_emit(buf, "%s\n", t7xx_mode_names[mode]); 120 121 return sysfs_emit(buf, "%s\n", t7xx_mode_names[T7XX_UNKNOWN]); 122 } 123 124 static DEVICE_ATTR_RW(t7xx_mode); 125 126 static ssize_t t7xx_debug_ports_store(struct device *dev, 127 struct device_attribute *attr, 128 const char *buf, size_t count) 129 { 130 struct t7xx_pci_dev *t7xx_dev; 131 struct pci_dev *pdev; 132 bool show; 133 int ret; 134 135 pdev = to_pci_dev(dev); 136 t7xx_dev = pci_get_drvdata(pdev); 137 if (!t7xx_dev) 138 return -ENODEV; 139 140 ret = kstrtobool(buf, &show); 141 if (ret < 0) 142 return ret; 143 144 t7xx_proxy_debug_ports_show(t7xx_dev, show); 145 WRITE_ONCE(t7xx_dev->debug_ports_show, show); 146 147 return count; 148 }; 149 150 static ssize_t t7xx_debug_ports_show(struct device *dev, 151 struct device_attribute *attr, 152 char *buf) 153 { 154 struct t7xx_pci_dev *t7xx_dev; 155 struct pci_dev *pdev; 156 bool show; 157 158 pdev = to_pci_dev(dev); 159 t7xx_dev = pci_get_drvdata(pdev); 160 if (!t7xx_dev) 161 return -ENODEV; 162 163 show = READ_ONCE(t7xx_dev->debug_ports_show); 164 165 return sysfs_emit(buf, "%d\n", show); 166 } 167 168 static DEVICE_ATTR_RW(t7xx_debug_ports); 169 170 static struct attribute *t7xx_attr[] = { 171 &dev_attr_t7xx_mode.attr, 172 &dev_attr_t7xx_debug_ports.attr, 173 NULL 174 }; 175 176 static const struct attribute_group t7xx_attribute_group = { 177 .attrs = t7xx_attr, 178 }; 179 180 void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode) 181 { 182 if (!t7xx_dev) 183 return; 184 185 WRITE_ONCE(t7xx_dev->mode, mode); 186 sysfs_notify(&t7xx_dev->pdev->dev.kobj, NULL, "t7xx_mode"); 187 } 188 189 enum t7xx_pm_state { 190 MTK_PM_EXCEPTION, 191 MTK_PM_INIT, /* Device initialized, but handshake not completed */ 192 MTK_PM_SUSPENDED, 193 MTK_PM_RESUMED, 194 }; 195 196 static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable) 197 { 198 void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL; 199 u32 value; 200 201 value = ioread32(ctrl_reg); 202 203 if (enable) 204 value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS; 205 else 206 value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS; 207 208 iowrite32(value, ctrl_reg); 209 } 210 211 static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev) 212 { 213 int ret, val; 214 215 ret = read_poll_timeout(ioread32, val, 216 (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK, 217 PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true, 218 IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); 219 if (ret == -ETIMEDOUT) 220 dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n"); 221 222 return ret; 223 } 224 225 static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev) 226 { 227 struct pci_dev *pdev = t7xx_dev->pdev; 228 229 INIT_LIST_HEAD(&t7xx_dev->md_pm_entities); 230 mutex_init(&t7xx_dev->md_pm_entity_mtx); 231 spin_lock_init(&t7xx_dev->md_pm_lock); 232 init_completion(&t7xx_dev->sleep_lock_acquire); 233 init_completion(&t7xx_dev->pm_sr_ack); 234 init_completion(&t7xx_dev->init_done); 235 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT); 236 237 device_init_wakeup(&pdev->dev, true); 238 dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags | 239 DPM_FLAG_NO_DIRECT_COMPLETE); 240 241 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 242 pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS); 243 pm_runtime_use_autosuspend(&pdev->dev); 244 245 return 0; 246 } 247 248 void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev) 249 { 250 /* Enable the PCIe resource lock only after MD deep sleep is done */ 251 t7xx_mhccif_mask_clr(t7xx_dev, 252 D2H_INT_DS_LOCK_ACK | 253 D2H_INT_SUSPEND_ACK | 254 D2H_INT_RESUME_ACK | 255 D2H_INT_SUSPEND_ACK_AP | 256 D2H_INT_RESUME_ACK_AP); 257 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 258 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 259 260 pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev); 261 pm_runtime_allow(&t7xx_dev->pdev->dev); 262 pm_runtime_put_noidle(&t7xx_dev->pdev->dev); 263 complete_all(&t7xx_dev->init_done); 264 } 265 266 static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev) 267 { 268 /* The device is kept in FSM re-init flow 269 * so just roll back PM setting to the init setting. 270 */ 271 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT); 272 273 pm_runtime_get_noresume(&t7xx_dev->pdev->dev); 274 275 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 276 return t7xx_wait_pm_config(t7xx_dev); 277 } 278 279 void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev) 280 { 281 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 282 t7xx_wait_pm_config(t7xx_dev); 283 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION); 284 } 285 286 int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) 287 { 288 struct md_pm_entity *entity; 289 290 mutex_lock(&t7xx_dev->md_pm_entity_mtx); 291 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 292 if (entity->id == pm_entity->id) { 293 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 294 return -EEXIST; 295 } 296 } 297 298 list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities); 299 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 300 return 0; 301 } 302 303 int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) 304 { 305 struct md_pm_entity *entity, *tmp_entity; 306 307 mutex_lock(&t7xx_dev->md_pm_entity_mtx); 308 list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) { 309 if (entity->id == pm_entity->id) { 310 list_del(&pm_entity->entity); 311 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 312 return 0; 313 } 314 } 315 316 mutex_unlock(&t7xx_dev->md_pm_entity_mtx); 317 318 return -ENXIO; 319 } 320 321 int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev) 322 { 323 struct device *dev = &t7xx_dev->pdev->dev; 324 int ret; 325 326 ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire, 327 msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS)); 328 if (!ret) 329 dev_err_ratelimited(dev, "Resource wait complete timed out\n"); 330 331 return ret; 332 } 333 334 /** 335 * t7xx_pci_disable_sleep() - Disable deep sleep capability. 336 * @t7xx_dev: MTK device. 337 * 338 * Lock the deep sleep capability, note that the device can still go into deep sleep 339 * state while device is in D0 state, from the host's point-of-view. 340 * 341 * If device is in deep sleep state, wake up the device and disable deep sleep capability. 342 */ 343 void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev) 344 { 345 unsigned long flags; 346 347 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); 348 t7xx_dev->sleep_disable_count++; 349 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED) 350 goto unlock_and_complete; 351 352 if (t7xx_dev->sleep_disable_count == 1) { 353 u32 status; 354 355 reinit_completion(&t7xx_dev->sleep_lock_acquire); 356 t7xx_dev_set_sleep_capability(t7xx_dev, false); 357 358 status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); 359 if (status & T7XX_PCIE_RESOURCE_STS_MSK) 360 goto unlock_and_complete; 361 362 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK); 363 } 364 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 365 return; 366 367 unlock_and_complete: 368 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 369 complete_all(&t7xx_dev->sleep_lock_acquire); 370 } 371 372 /** 373 * t7xx_pci_enable_sleep() - Enable deep sleep capability. 374 * @t7xx_dev: MTK device. 375 * 376 * After enabling deep sleep, device can enter into deep sleep state. 377 */ 378 void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev) 379 { 380 unsigned long flags; 381 382 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); 383 t7xx_dev->sleep_disable_count--; 384 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED) 385 goto unlock; 386 387 if (t7xx_dev->sleep_disable_count == 0) 388 t7xx_dev_set_sleep_capability(t7xx_dev, true); 389 390 unlock: 391 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags); 392 } 393 394 static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request) 395 { 396 unsigned long wait_ret; 397 398 reinit_completion(&t7xx_dev->pm_sr_ack); 399 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request); 400 wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack, 401 msecs_to_jiffies(PM_ACK_TIMEOUT_MS)); 402 if (!wait_ret) 403 return -ETIMEDOUT; 404 405 return 0; 406 } 407 408 static int __t7xx_pci_pm_suspend(struct pci_dev *pdev) 409 { 410 enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID; 411 struct t7xx_pci_dev *t7xx_dev; 412 struct md_pm_entity *entity; 413 int ret; 414 415 t7xx_dev = pci_get_drvdata(pdev); 416 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT || 417 READ_ONCE(t7xx_dev->mode) != T7XX_READY) { 418 dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n"); 419 return -EFAULT; 420 } 421 422 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 423 ret = t7xx_wait_pm_config(t7xx_dev); 424 if (ret) { 425 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 426 return ret; 427 } 428 429 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 430 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT); 431 t7xx_dev->rgu_pci_irq_en = false; 432 433 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 434 if (!entity->suspend) 435 continue; 436 437 ret = entity->suspend(t7xx_dev, entity->entity_param); 438 if (ret) { 439 entity_id = entity->id; 440 dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id); 441 goto abort_suspend; 442 } 443 } 444 445 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ); 446 if (ret) { 447 dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret); 448 goto abort_suspend; 449 } 450 451 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP); 452 if (ret) { 453 t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); 454 dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret); 455 goto abort_suspend; 456 } 457 458 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 459 if (entity->suspend_late) 460 entity->suspend_late(t7xx_dev, entity->entity_param); 461 } 462 463 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 464 return 0; 465 466 abort_suspend: 467 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 468 if (entity_id == entity->id) 469 break; 470 471 if (entity->resume) 472 entity->resume(t7xx_dev, entity->entity_param); 473 } 474 475 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 476 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 477 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 478 return ret; 479 } 480 481 static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev) 482 { 483 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); 484 485 /* Disable interrupt first and let the IPs enable them */ 486 iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0); 487 488 /* Device disables PCIe interrupts during resume and 489 * following function will re-enable PCIe interrupts. 490 */ 491 t7xx_pcie_mac_interrupts_en(t7xx_dev); 492 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT); 493 } 494 495 static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3) 496 { 497 int ret; 498 499 ret = pcim_enable_device(t7xx_dev->pdev); 500 if (ret) 501 return ret; 502 503 t7xx_pcie_mac_atr_init(t7xx_dev); 504 t7xx_pcie_interrupt_reinit(t7xx_dev); 505 506 if (is_d3) { 507 t7xx_mhccif_init(t7xx_dev); 508 t7xx_pci_pm_reinit(t7xx_dev); 509 } 510 511 return 0; 512 } 513 514 static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event) 515 { 516 struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl; 517 struct device *dev = &t7xx_dev->pdev->dev; 518 int ret = -EINVAL; 519 520 switch (event) { 521 case FSM_CMD_STOP: 522 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION); 523 break; 524 525 case FSM_CMD_START: 526 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT); 527 t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT); 528 t7xx_dev->rgu_pci_irq_en = true; 529 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 530 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0); 531 break; 532 533 default: 534 break; 535 } 536 537 if (ret) 538 dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret); 539 540 return ret; 541 } 542 543 int t7xx_pci_reprobe_early(struct t7xx_pci_dev *t7xx_dev) 544 { 545 enum t7xx_mode mode = READ_ONCE(t7xx_dev->mode); 546 int ret; 547 548 if (mode == T7XX_FASTBOOT_DOWNLOAD) 549 pm_runtime_put_noidle(&t7xx_dev->pdev->dev); 550 551 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP); 552 if (ret) 553 return ret; 554 555 return 0; 556 } 557 558 int t7xx_pci_reprobe(struct t7xx_pci_dev *t7xx_dev, bool boot) 559 { 560 int ret; 561 562 ret = t7xx_pcie_reinit(t7xx_dev, boot); 563 if (ret) 564 return ret; 565 566 t7xx_clear_rgu_irq(t7xx_dev); 567 return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START); 568 } 569 570 static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check) 571 { 572 struct t7xx_pci_dev *t7xx_dev; 573 struct md_pm_entity *entity; 574 u32 prev_state; 575 int ret = 0; 576 577 t7xx_dev = pci_get_drvdata(pdev); 578 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) { 579 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 580 return 0; 581 } 582 583 t7xx_pcie_mac_interrupts_en(t7xx_dev); 584 prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE); 585 586 if (state_check) { 587 /* For D3/L3 resume, the device could boot so quickly that the 588 * initial value of the dummy register might be overwritten. 589 * Identify new boots if the ATR source address register is not initialized. 590 */ 591 u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) + 592 ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR); 593 if (prev_state == PM_RESUME_REG_STATE_L3 || 594 (prev_state == PM_RESUME_REG_STATE_INIT && 595 atr_reg_val == ATR_SRC_ADDR_INVALID)) { 596 ret = t7xx_pci_reprobe_early(t7xx_dev); 597 if (ret) 598 return ret; 599 600 return t7xx_pci_reprobe(t7xx_dev, true); 601 } 602 603 if (prev_state == PM_RESUME_REG_STATE_EXP || 604 prev_state == PM_RESUME_REG_STATE_L2_EXP) { 605 if (prev_state == PM_RESUME_REG_STATE_L2_EXP) { 606 ret = t7xx_pcie_reinit(t7xx_dev, false); 607 if (ret) 608 return ret; 609 } 610 611 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 612 t7xx_dev->rgu_pci_irq_en = true; 613 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 614 615 t7xx_mhccif_mask_clr(t7xx_dev, 616 D2H_INT_EXCEPTION_INIT | 617 D2H_INT_EXCEPTION_INIT_DONE | 618 D2H_INT_EXCEPTION_CLEARQ_DONE | 619 D2H_INT_EXCEPTION_ALLQ_RESET | 620 D2H_INT_PORT_ENUM); 621 622 return ret; 623 } 624 625 if (prev_state == PM_RESUME_REG_STATE_L2) { 626 ret = t7xx_pcie_reinit(t7xx_dev, false); 627 if (ret) 628 return ret; 629 630 } else if (prev_state != PM_RESUME_REG_STATE_L1 && 631 prev_state != PM_RESUME_REG_STATE_INIT) { 632 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP); 633 if (ret) 634 return ret; 635 636 t7xx_clear_rgu_irq(t7xx_dev); 637 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED); 638 return 0; 639 } 640 } 641 642 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); 643 t7xx_wait_pm_config(t7xx_dev); 644 645 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 646 if (entity->resume_early) 647 entity->resume_early(t7xx_dev, entity->entity_param); 648 } 649 650 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); 651 if (ret) 652 dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret); 653 654 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP); 655 if (ret) 656 dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret); 657 658 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { 659 if (entity->resume) { 660 ret = entity->resume(t7xx_dev, entity->entity_param); 661 if (ret) 662 dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n", 663 entity->id, ret); 664 } 665 } 666 667 t7xx_dev->rgu_pci_irq_en = true; 668 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT); 669 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); 670 pm_runtime_mark_last_busy(&pdev->dev); 671 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED); 672 673 return ret; 674 } 675 676 static int t7xx_pci_pm_resume_noirq(struct device *dev) 677 { 678 struct pci_dev *pdev = to_pci_dev(dev); 679 struct t7xx_pci_dev *t7xx_dev; 680 681 t7xx_dev = pci_get_drvdata(pdev); 682 t7xx_pcie_mac_interrupts_dis(t7xx_dev); 683 684 return 0; 685 } 686 687 static void t7xx_pci_shutdown(struct pci_dev *pdev) 688 { 689 __t7xx_pci_pm_suspend(pdev); 690 } 691 692 static int t7xx_pci_pm_prepare(struct device *dev) 693 { 694 struct pci_dev *pdev = to_pci_dev(dev); 695 struct t7xx_pci_dev *t7xx_dev; 696 697 t7xx_dev = pci_get_drvdata(pdev); 698 if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) { 699 dev_warn(dev, "Not ready for system sleep.\n"); 700 return -ETIMEDOUT; 701 } 702 703 return 0; 704 } 705 706 static int t7xx_pci_pm_suspend(struct device *dev) 707 { 708 return __t7xx_pci_pm_suspend(to_pci_dev(dev)); 709 } 710 711 static int t7xx_pci_pm_resume(struct device *dev) 712 { 713 return __t7xx_pci_pm_resume(to_pci_dev(dev), true); 714 } 715 716 static int t7xx_pci_pm_thaw(struct device *dev) 717 { 718 return __t7xx_pci_pm_resume(to_pci_dev(dev), false); 719 } 720 721 static int t7xx_pci_pm_runtime_suspend(struct device *dev) 722 { 723 return __t7xx_pci_pm_suspend(to_pci_dev(dev)); 724 } 725 726 static int t7xx_pci_pm_runtime_resume(struct device *dev) 727 { 728 return __t7xx_pci_pm_resume(to_pci_dev(dev), true); 729 } 730 731 static const struct dev_pm_ops t7xx_pci_pm_ops = { 732 .prepare = t7xx_pci_pm_prepare, 733 .suspend = t7xx_pci_pm_suspend, 734 .resume = t7xx_pci_pm_resume, 735 .resume_noirq = t7xx_pci_pm_resume_noirq, 736 .freeze = t7xx_pci_pm_suspend, 737 .thaw = t7xx_pci_pm_thaw, 738 .poweroff = t7xx_pci_pm_suspend, 739 .restore = t7xx_pci_pm_resume, 740 .restore_noirq = t7xx_pci_pm_resume_noirq, 741 .runtime_suspend = t7xx_pci_pm_runtime_suspend, 742 .runtime_resume = t7xx_pci_pm_runtime_resume 743 }; 744 745 static int t7xx_request_irq(struct pci_dev *pdev) 746 { 747 struct t7xx_pci_dev *t7xx_dev; 748 int ret = 0, i; 749 750 t7xx_dev = pci_get_drvdata(pdev); 751 752 for (i = 0; i < EXT_INT_NUM; i++) { 753 const char *irq_descr; 754 int irq_vec; 755 756 if (!t7xx_dev->intr_handler[i]) 757 continue; 758 759 irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d", 760 dev_driver_string(&pdev->dev), i); 761 if (!irq_descr) { 762 ret = -ENOMEM; 763 break; 764 } 765 766 irq_vec = pci_irq_vector(pdev, i); 767 ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i], 768 t7xx_dev->intr_thread[i], 0, irq_descr, 769 t7xx_dev->callback_param[i]); 770 if (ret) { 771 dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret); 772 break; 773 } 774 } 775 776 if (ret) { 777 while (i--) { 778 if (!t7xx_dev->intr_handler[i]) 779 continue; 780 781 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]); 782 } 783 } 784 785 return ret; 786 } 787 788 static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev) 789 { 790 struct pci_dev *pdev = t7xx_dev->pdev; 791 int ret; 792 793 /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */ 794 ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX); 795 if (ret < 0) { 796 dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret); 797 return ret; 798 } 799 800 ret = t7xx_request_irq(pdev); 801 if (ret) { 802 pci_free_irq_vectors(pdev); 803 return ret; 804 } 805 806 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); 807 return 0; 808 } 809 810 static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev) 811 { 812 int ret, i; 813 814 if (!t7xx_dev->pdev->msix_cap) 815 return -EINVAL; 816 817 ret = t7xx_setup_msix(t7xx_dev); 818 if (ret) 819 return ret; 820 821 /* IPs enable interrupts when ready */ 822 for (i = 0; i < EXT_INT_NUM; i++) 823 t7xx_pcie_mac_set_int(t7xx_dev, i); 824 825 return 0; 826 } 827 828 static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev) 829 { 830 t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base + 831 INFRACFG_AO_DEV_CHIP - 832 t7xx_dev->base_addr.pcie_dev_reg_trsl_addr; 833 } 834 835 static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 836 { 837 struct t7xx_pci_dev *t7xx_dev; 838 void __iomem *iomem; 839 int ret; 840 841 t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL); 842 if (!t7xx_dev) 843 return -ENOMEM; 844 845 pci_set_drvdata(pdev, t7xx_dev); 846 t7xx_dev->pdev = pdev; 847 848 ret = pcim_enable_device(pdev); 849 if (ret) 850 return ret; 851 852 pci_set_master(pdev); 853 854 iomem = pcim_iomap_region(pdev, T7XX_PCI_IREG_BASE, DRIVER_NAME); 855 ret = PTR_ERR_OR_ZERO(iomem); 856 if (ret) { 857 dev_err(&pdev->dev, "Could not request IREG BAR: %d\n", ret); 858 return -ENOMEM; 859 } 860 IREG_BASE(t7xx_dev) = iomem; 861 862 iomem = pcim_iomap_region(pdev, T7XX_PCI_EREG_BASE, DRIVER_NAME); 863 ret = PTR_ERR_OR_ZERO(iomem); 864 if (ret) { 865 dev_err(&pdev->dev, "Could not request EREG BAR: %d\n", ret); 866 return -ENOMEM; 867 } 868 t7xx_dev->base_addr.pcie_ext_reg_base = iomem; 869 870 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 871 if (ret) { 872 dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret); 873 return ret; 874 } 875 876 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 877 if (ret) { 878 dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret); 879 return ret; 880 } 881 882 ret = t7xx_pci_pm_init(t7xx_dev); 883 if (ret) 884 return ret; 885 886 t7xx_pcie_mac_atr_init(t7xx_dev); 887 t7xx_pci_infracfg_ao_calc(t7xx_dev); 888 t7xx_mhccif_init(t7xx_dev); 889 890 ret = t7xx_md_init(t7xx_dev); 891 if (ret) 892 return ret; 893 894 t7xx_pcie_mac_interrupts_dis(t7xx_dev); 895 896 ret = sysfs_create_group(&t7xx_dev->pdev->dev.kobj, 897 &t7xx_attribute_group); 898 if (ret) 899 goto err_md_exit; 900 901 ret = t7xx_interrupt_init(t7xx_dev); 902 if (ret) 903 goto err_remove_group; 904 905 906 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT); 907 t7xx_pcie_mac_interrupts_en(t7xx_dev); 908 909 return 0; 910 911 err_remove_group: 912 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj, 913 &t7xx_attribute_group); 914 915 err_md_exit: 916 t7xx_md_exit(t7xx_dev); 917 return ret; 918 } 919 920 static void t7xx_pci_remove(struct pci_dev *pdev) 921 { 922 struct t7xx_pci_dev *t7xx_dev; 923 int i; 924 925 t7xx_dev = pci_get_drvdata(pdev); 926 927 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj, 928 &t7xx_attribute_group); 929 t7xx_md_exit(t7xx_dev); 930 931 for (i = 0; i < EXT_INT_NUM; i++) { 932 if (!t7xx_dev->intr_handler[i]) 933 continue; 934 935 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]); 936 } 937 938 pci_free_irq_vectors(t7xx_dev->pdev); 939 } 940 941 static const struct pci_device_id t7xx_pci_table[] = { 942 { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) }, 943 { PCI_DEVICE(0x14c0, 0x4d75) }, // Dell DW5933e 944 { } 945 }; 946 MODULE_DEVICE_TABLE(pci, t7xx_pci_table); 947 948 static struct pci_driver t7xx_pci_driver = { 949 .name = DRIVER_NAME, 950 .id_table = t7xx_pci_table, 951 .probe = t7xx_pci_probe, 952 .remove = t7xx_pci_remove, 953 .driver.pm = &t7xx_pci_pm_ops, 954 .shutdown = t7xx_pci_shutdown, 955 }; 956 957 module_pci_driver(t7xx_pci_driver); 958 959 MODULE_AUTHOR("MediaTek Inc"); 960 MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver"); 961 MODULE_LICENSE("GPL"); 962