1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2021, MediaTek Inc.
4 * Copyright (c) 2021-2022, Intel Corporation.
5 *
6 * Authors:
7 * Haijun Liu <haijun.liu@mediatek.com>
8 * Ricardo Martinez <ricardo.martinez@linux.intel.com>
9 * Sreehari Kancharla <sreehari.kancharla@intel.com>
10 *
11 * Contributors:
12 * Amir Hanania <amir.hanania@intel.com>
13 * Andy Shevchenko <andriy.shevchenko@linux.intel.com>
14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
15 * Eliot Lee <eliot.lee@intel.com>
16 * Moises Veleta <moises.veleta@intel.com>
17 */
18
19 #include <linux/atomic.h>
20 #include <linux/bits.h>
21 #include <linux/completion.h>
22 #include <linux/device.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/gfp.h>
25 #include <linux/interrupt.h>
26 #include <linux/io.h>
27 #include <linux/iopoll.h>
28 #include <linux/jiffies.h>
29 #include <linux/list.h>
30 #include <linux/module.h>
31 #include <linux/mutex.h>
32 #include <linux/pci.h>
33 #include <linux/pm.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/spinlock.h>
36
37 #include "t7xx_mhccif.h"
38 #include "t7xx_modem_ops.h"
39 #include "t7xx_pci.h"
40 #include "t7xx_pcie_mac.h"
41 #include "t7xx_reg.h"
42 #include "t7xx_state_monitor.h"
43 #include "t7xx_port_proxy.h"
44
45 #define DRIVER_NAME "mtk_t7xx"
46
47 #define T7XX_PCI_IREG_BASE 0
48 #define T7XX_PCI_EREG_BASE 2
49
50 #define T7XX_INIT_TIMEOUT 20
51 #define PM_SLEEP_DIS_TIMEOUT_MS 20
52 #define PM_ACK_TIMEOUT_MS 1500
53 #define PM_AUTOSUSPEND_MS 5000
54 #define PM_RESOURCE_POLL_TIMEOUT_US 10000
55 #define PM_RESOURCE_POLL_STEP_US 100
56
57 static const char * const t7xx_mode_names[] = {
58 [T7XX_UNKNOWN] = "unknown",
59 [T7XX_READY] = "ready",
60 [T7XX_RESET] = "reset",
61 [T7XX_FASTBOOT_SWITCHING] = "fastboot_switching",
62 [T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download",
63 [T7XX_FASTBOOT_DUMP] = "fastboot_dump",
64 };
65
66 static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST);
67
t7xx_mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)68 static ssize_t t7xx_mode_store(struct device *dev,
69 struct device_attribute *attr,
70 const char *buf, size_t count)
71 {
72 struct t7xx_pci_dev *t7xx_dev;
73 struct pci_dev *pdev;
74 enum t7xx_mode mode;
75 int index = 0;
76
77 pdev = to_pci_dev(dev);
78 t7xx_dev = pci_get_drvdata(pdev);
79 if (!t7xx_dev)
80 return -ENODEV;
81
82 mode = READ_ONCE(t7xx_dev->mode);
83
84 index = sysfs_match_string(t7xx_mode_names, buf);
85 if (index == mode)
86 return -EBUSY;
87
88 if (index == T7XX_FASTBOOT_SWITCHING) {
89 if (mode == T7XX_FASTBOOT_DOWNLOAD)
90 return count;
91
92 WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING);
93 pm_runtime_resume(dev);
94 t7xx_reset_device(t7xx_dev, FASTBOOT);
95 } else if (index == T7XX_RESET) {
96 pm_runtime_resume(dev);
97 t7xx_reset_device(t7xx_dev, PLDR);
98 }
99
100 return count;
101 };
102
t7xx_mode_show(struct device * dev,struct device_attribute * attr,char * buf)103 static ssize_t t7xx_mode_show(struct device *dev,
104 struct device_attribute *attr,
105 char *buf)
106 {
107 enum t7xx_mode mode = T7XX_UNKNOWN;
108 struct t7xx_pci_dev *t7xx_dev;
109 struct pci_dev *pdev;
110
111 pdev = to_pci_dev(dev);
112 t7xx_dev = pci_get_drvdata(pdev);
113 if (!t7xx_dev)
114 return -ENODEV;
115
116 mode = READ_ONCE(t7xx_dev->mode);
117 if (mode < T7XX_MODE_LAST)
118 return sysfs_emit(buf, "%s\n", t7xx_mode_names[mode]);
119
120 return sysfs_emit(buf, "%s\n", t7xx_mode_names[T7XX_UNKNOWN]);
121 }
122
123 static DEVICE_ATTR_RW(t7xx_mode);
124
t7xx_debug_ports_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)125 static ssize_t t7xx_debug_ports_store(struct device *dev,
126 struct device_attribute *attr,
127 const char *buf, size_t count)
128 {
129 struct t7xx_pci_dev *t7xx_dev;
130 struct pci_dev *pdev;
131 bool show;
132 int ret;
133
134 pdev = to_pci_dev(dev);
135 t7xx_dev = pci_get_drvdata(pdev);
136 if (!t7xx_dev)
137 return -ENODEV;
138
139 ret = kstrtobool(buf, &show);
140 if (ret < 0)
141 return ret;
142
143 t7xx_proxy_debug_ports_show(t7xx_dev, show);
144 WRITE_ONCE(t7xx_dev->debug_ports_show, show);
145
146 return count;
147 };
148
t7xx_debug_ports_show(struct device * dev,struct device_attribute * attr,char * buf)149 static ssize_t t7xx_debug_ports_show(struct device *dev,
150 struct device_attribute *attr,
151 char *buf)
152 {
153 struct t7xx_pci_dev *t7xx_dev;
154 struct pci_dev *pdev;
155 bool show;
156
157 pdev = to_pci_dev(dev);
158 t7xx_dev = pci_get_drvdata(pdev);
159 if (!t7xx_dev)
160 return -ENODEV;
161
162 show = READ_ONCE(t7xx_dev->debug_ports_show);
163
164 return sysfs_emit(buf, "%d\n", show);
165 }
166
167 static DEVICE_ATTR_RW(t7xx_debug_ports);
168
169 static struct attribute *t7xx_attr[] = {
170 &dev_attr_t7xx_mode.attr,
171 &dev_attr_t7xx_debug_ports.attr,
172 NULL
173 };
174
175 static const struct attribute_group t7xx_attribute_group = {
176 .attrs = t7xx_attr,
177 };
178
t7xx_mode_update(struct t7xx_pci_dev * t7xx_dev,enum t7xx_mode mode)179 void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode)
180 {
181 if (!t7xx_dev)
182 return;
183
184 WRITE_ONCE(t7xx_dev->mode, mode);
185 sysfs_notify(&t7xx_dev->pdev->dev.kobj, NULL, "t7xx_mode");
186 }
187
188 enum t7xx_pm_state {
189 MTK_PM_EXCEPTION,
190 MTK_PM_INIT, /* Device initialized, but handshake not completed */
191 MTK_PM_SUSPENDED,
192 MTK_PM_RESUMED,
193 };
194
t7xx_dev_set_sleep_capability(struct t7xx_pci_dev * t7xx_dev,bool enable)195 static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable)
196 {
197 void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL;
198 u32 value;
199
200 value = ioread32(ctrl_reg);
201
202 if (enable)
203 value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS;
204 else
205 value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS;
206
207 iowrite32(value, ctrl_reg);
208 }
209
t7xx_wait_pm_config(struct t7xx_pci_dev * t7xx_dev)210 static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev)
211 {
212 int ret, val;
213
214 ret = read_poll_timeout(ioread32, val,
215 (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK,
216 PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true,
217 IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
218 if (ret == -ETIMEDOUT)
219 dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n");
220
221 return ret;
222 }
223
t7xx_pci_pm_init(struct t7xx_pci_dev * t7xx_dev)224 static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev)
225 {
226 struct pci_dev *pdev = t7xx_dev->pdev;
227
228 INIT_LIST_HEAD(&t7xx_dev->md_pm_entities);
229 mutex_init(&t7xx_dev->md_pm_entity_mtx);
230 spin_lock_init(&t7xx_dev->md_pm_lock);
231 init_completion(&t7xx_dev->sleep_lock_acquire);
232 init_completion(&t7xx_dev->pm_sr_ack);
233 init_completion(&t7xx_dev->init_done);
234 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
235
236 device_init_wakeup(&pdev->dev, true);
237 dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags |
238 DPM_FLAG_NO_DIRECT_COMPLETE);
239
240 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
241 pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS);
242 pm_runtime_use_autosuspend(&pdev->dev);
243
244 return 0;
245 }
246
t7xx_pci_pm_init_late(struct t7xx_pci_dev * t7xx_dev)247 void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev)
248 {
249 /* Enable the PCIe resource lock only after MD deep sleep is done */
250 t7xx_mhccif_mask_clr(t7xx_dev,
251 D2H_INT_DS_LOCK_ACK |
252 D2H_INT_SUSPEND_ACK |
253 D2H_INT_RESUME_ACK |
254 D2H_INT_SUSPEND_ACK_AP |
255 D2H_INT_RESUME_ACK_AP);
256 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
257 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
258
259 pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev);
260 pm_runtime_allow(&t7xx_dev->pdev->dev);
261 pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
262 complete_all(&t7xx_dev->init_done);
263 }
264
t7xx_pci_pm_reinit(struct t7xx_pci_dev * t7xx_dev)265 static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev)
266 {
267 /* The device is kept in FSM re-init flow
268 * so just roll back PM setting to the init setting.
269 */
270 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
271
272 pm_runtime_get_noresume(&t7xx_dev->pdev->dev);
273
274 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
275 return t7xx_wait_pm_config(t7xx_dev);
276 }
277
t7xx_pci_pm_exp_detected(struct t7xx_pci_dev * t7xx_dev)278 void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev)
279 {
280 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
281 t7xx_wait_pm_config(t7xx_dev);
282 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION);
283 }
284
t7xx_pci_pm_entity_register(struct t7xx_pci_dev * t7xx_dev,struct md_pm_entity * pm_entity)285 int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
286 {
287 struct md_pm_entity *entity;
288
289 mutex_lock(&t7xx_dev->md_pm_entity_mtx);
290 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
291 if (entity->id == pm_entity->id) {
292 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
293 return -EEXIST;
294 }
295 }
296
297 list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities);
298 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
299 return 0;
300 }
301
t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev * t7xx_dev,struct md_pm_entity * pm_entity)302 int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
303 {
304 struct md_pm_entity *entity, *tmp_entity;
305
306 mutex_lock(&t7xx_dev->md_pm_entity_mtx);
307 list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) {
308 if (entity->id == pm_entity->id) {
309 list_del(&pm_entity->entity);
310 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
311 return 0;
312 }
313 }
314
315 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
316
317 return -ENXIO;
318 }
319
t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev * t7xx_dev)320 int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev)
321 {
322 struct device *dev = &t7xx_dev->pdev->dev;
323 int ret;
324
325 ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire,
326 msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS));
327 if (!ret)
328 dev_err_ratelimited(dev, "Resource wait complete timed out\n");
329
330 return ret;
331 }
332
333 /**
334 * t7xx_pci_disable_sleep() - Disable deep sleep capability.
335 * @t7xx_dev: MTK device.
336 *
337 * Lock the deep sleep capability, note that the device can still go into deep sleep
338 * state while device is in D0 state, from the host's point-of-view.
339 *
340 * If device is in deep sleep state, wake up the device and disable deep sleep capability.
341 */
t7xx_pci_disable_sleep(struct t7xx_pci_dev * t7xx_dev)342 void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev)
343 {
344 unsigned long flags;
345
346 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
347 t7xx_dev->sleep_disable_count++;
348 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
349 goto unlock_and_complete;
350
351 if (t7xx_dev->sleep_disable_count == 1) {
352 u32 status;
353
354 reinit_completion(&t7xx_dev->sleep_lock_acquire);
355 t7xx_dev_set_sleep_capability(t7xx_dev, false);
356
357 status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
358 if (status & T7XX_PCIE_RESOURCE_STS_MSK)
359 goto unlock_and_complete;
360
361 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK);
362 }
363 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
364 return;
365
366 unlock_and_complete:
367 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
368 complete_all(&t7xx_dev->sleep_lock_acquire);
369 }
370
371 /**
372 * t7xx_pci_enable_sleep() - Enable deep sleep capability.
373 * @t7xx_dev: MTK device.
374 *
375 * After enabling deep sleep, device can enter into deep sleep state.
376 */
t7xx_pci_enable_sleep(struct t7xx_pci_dev * t7xx_dev)377 void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev)
378 {
379 unsigned long flags;
380
381 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
382 t7xx_dev->sleep_disable_count--;
383 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
384 goto unlock;
385
386 if (t7xx_dev->sleep_disable_count == 0)
387 t7xx_dev_set_sleep_capability(t7xx_dev, true);
388
389 unlock:
390 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
391 }
392
t7xx_send_pm_request(struct t7xx_pci_dev * t7xx_dev,u32 request)393 static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request)
394 {
395 unsigned long wait_ret;
396
397 reinit_completion(&t7xx_dev->pm_sr_ack);
398 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request);
399 wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack,
400 msecs_to_jiffies(PM_ACK_TIMEOUT_MS));
401 if (!wait_ret)
402 return -ETIMEDOUT;
403
404 return 0;
405 }
406
__t7xx_pci_pm_suspend(struct pci_dev * pdev)407 static int __t7xx_pci_pm_suspend(struct pci_dev *pdev)
408 {
409 enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID;
410 struct t7xx_pci_dev *t7xx_dev;
411 struct md_pm_entity *entity;
412 int ret;
413
414 t7xx_dev = pci_get_drvdata(pdev);
415 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT ||
416 READ_ONCE(t7xx_dev->mode) != T7XX_READY) {
417 dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n");
418 return -EFAULT;
419 }
420
421 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
422 ret = t7xx_wait_pm_config(t7xx_dev);
423 if (ret) {
424 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
425 return ret;
426 }
427
428 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
429 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
430 t7xx_dev->rgu_pci_irq_en = false;
431
432 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
433 if (!entity->suspend)
434 continue;
435
436 ret = entity->suspend(t7xx_dev, entity->entity_param);
437 if (ret) {
438 entity_id = entity->id;
439 dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id);
440 goto abort_suspend;
441 }
442 }
443
444 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ);
445 if (ret) {
446 dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret);
447 goto abort_suspend;
448 }
449
450 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP);
451 if (ret) {
452 t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
453 dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret);
454 goto abort_suspend;
455 }
456
457 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
458 if (entity->suspend_late)
459 entity->suspend_late(t7xx_dev, entity->entity_param);
460 }
461
462 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
463 return 0;
464
465 abort_suspend:
466 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
467 if (entity_id == entity->id)
468 break;
469
470 if (entity->resume)
471 entity->resume(t7xx_dev, entity->entity_param);
472 }
473
474 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
475 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
476 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
477 return ret;
478 }
479
t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev * t7xx_dev)480 static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev)
481 {
482 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
483
484 /* Disable interrupt first and let the IPs enable them */
485 iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0);
486
487 /* Device disables PCIe interrupts during resume and
488 * following function will re-enable PCIe interrupts.
489 */
490 t7xx_pcie_mac_interrupts_en(t7xx_dev);
491 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
492 }
493
t7xx_pcie_reinit(struct t7xx_pci_dev * t7xx_dev,bool is_d3)494 static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3)
495 {
496 int ret;
497
498 ret = pcim_enable_device(t7xx_dev->pdev);
499 if (ret)
500 return ret;
501
502 t7xx_pcie_mac_atr_init(t7xx_dev);
503 t7xx_pcie_interrupt_reinit(t7xx_dev);
504
505 if (is_d3) {
506 t7xx_mhccif_init(t7xx_dev);
507 t7xx_pci_pm_reinit(t7xx_dev);
508 }
509
510 return 0;
511 }
512
t7xx_send_fsm_command(struct t7xx_pci_dev * t7xx_dev,u32 event)513 static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event)
514 {
515 struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl;
516 struct device *dev = &t7xx_dev->pdev->dev;
517 int ret = -EINVAL;
518
519 switch (event) {
520 case FSM_CMD_STOP:
521 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION);
522 break;
523
524 case FSM_CMD_START:
525 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
526 t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT);
527 t7xx_dev->rgu_pci_irq_en = true;
528 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
529 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0);
530 break;
531
532 default:
533 break;
534 }
535
536 if (ret)
537 dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret);
538
539 return ret;
540 }
541
t7xx_pci_reprobe_early(struct t7xx_pci_dev * t7xx_dev)542 int t7xx_pci_reprobe_early(struct t7xx_pci_dev *t7xx_dev)
543 {
544 enum t7xx_mode mode = READ_ONCE(t7xx_dev->mode);
545 int ret;
546
547 if (mode == T7XX_FASTBOOT_DOWNLOAD)
548 pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
549
550 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
551 if (ret)
552 return ret;
553
554 return 0;
555 }
556
t7xx_pci_reprobe(struct t7xx_pci_dev * t7xx_dev,bool boot)557 int t7xx_pci_reprobe(struct t7xx_pci_dev *t7xx_dev, bool boot)
558 {
559 int ret;
560
561 ret = t7xx_pcie_reinit(t7xx_dev, boot);
562 if (ret)
563 return ret;
564
565 t7xx_clear_rgu_irq(t7xx_dev);
566 return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START);
567 }
568
__t7xx_pci_pm_resume(struct pci_dev * pdev,bool state_check)569 static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check)
570 {
571 struct t7xx_pci_dev *t7xx_dev;
572 struct md_pm_entity *entity;
573 u32 prev_state;
574 int ret = 0;
575
576 t7xx_dev = pci_get_drvdata(pdev);
577 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) {
578 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
579 return 0;
580 }
581
582 t7xx_pcie_mac_interrupts_en(t7xx_dev);
583 prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE);
584
585 if (state_check) {
586 /* For D3/L3 resume, the device could boot so quickly that the
587 * initial value of the dummy register might be overwritten.
588 * Identify new boots if the ATR source address register is not initialized.
589 */
590 u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) +
591 ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR);
592 if (prev_state == PM_RESUME_REG_STATE_L3 ||
593 (prev_state == PM_RESUME_REG_STATE_INIT &&
594 atr_reg_val == ATR_SRC_ADDR_INVALID)) {
595 ret = t7xx_pci_reprobe_early(t7xx_dev);
596 if (ret)
597 return ret;
598
599 return t7xx_pci_reprobe(t7xx_dev, true);
600 }
601
602 if (prev_state == PM_RESUME_REG_STATE_EXP ||
603 prev_state == PM_RESUME_REG_STATE_L2_EXP) {
604 if (prev_state == PM_RESUME_REG_STATE_L2_EXP) {
605 ret = t7xx_pcie_reinit(t7xx_dev, false);
606 if (ret)
607 return ret;
608 }
609
610 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
611 t7xx_dev->rgu_pci_irq_en = true;
612 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
613
614 t7xx_mhccif_mask_clr(t7xx_dev,
615 D2H_INT_EXCEPTION_INIT |
616 D2H_INT_EXCEPTION_INIT_DONE |
617 D2H_INT_EXCEPTION_CLEARQ_DONE |
618 D2H_INT_EXCEPTION_ALLQ_RESET |
619 D2H_INT_PORT_ENUM);
620
621 return ret;
622 }
623
624 if (prev_state == PM_RESUME_REG_STATE_L2) {
625 ret = t7xx_pcie_reinit(t7xx_dev, false);
626 if (ret)
627 return ret;
628
629 } else if (prev_state != PM_RESUME_REG_STATE_L1 &&
630 prev_state != PM_RESUME_REG_STATE_INIT) {
631 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
632 if (ret)
633 return ret;
634
635 t7xx_clear_rgu_irq(t7xx_dev);
636 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
637 return 0;
638 }
639 }
640
641 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
642 t7xx_wait_pm_config(t7xx_dev);
643
644 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
645 if (entity->resume_early)
646 entity->resume_early(t7xx_dev, entity->entity_param);
647 }
648
649 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
650 if (ret)
651 dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret);
652
653 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP);
654 if (ret)
655 dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret);
656
657 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
658 if (entity->resume) {
659 ret = entity->resume(t7xx_dev, entity->entity_param);
660 if (ret)
661 dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n",
662 entity->id, ret);
663 }
664 }
665
666 t7xx_dev->rgu_pci_irq_en = true;
667 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
668 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
669 pm_runtime_mark_last_busy(&pdev->dev);
670 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
671
672 return ret;
673 }
674
t7xx_pci_pm_resume_noirq(struct device * dev)675 static int t7xx_pci_pm_resume_noirq(struct device *dev)
676 {
677 struct pci_dev *pdev = to_pci_dev(dev);
678 struct t7xx_pci_dev *t7xx_dev;
679
680 t7xx_dev = pci_get_drvdata(pdev);
681 t7xx_pcie_mac_interrupts_dis(t7xx_dev);
682
683 return 0;
684 }
685
t7xx_pci_shutdown(struct pci_dev * pdev)686 static void t7xx_pci_shutdown(struct pci_dev *pdev)
687 {
688 __t7xx_pci_pm_suspend(pdev);
689 }
690
t7xx_pci_pm_prepare(struct device * dev)691 static int t7xx_pci_pm_prepare(struct device *dev)
692 {
693 struct pci_dev *pdev = to_pci_dev(dev);
694 struct t7xx_pci_dev *t7xx_dev;
695
696 t7xx_dev = pci_get_drvdata(pdev);
697 if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) {
698 dev_warn(dev, "Not ready for system sleep.\n");
699 return -ETIMEDOUT;
700 }
701
702 return 0;
703 }
704
t7xx_pci_pm_suspend(struct device * dev)705 static int t7xx_pci_pm_suspend(struct device *dev)
706 {
707 return __t7xx_pci_pm_suspend(to_pci_dev(dev));
708 }
709
t7xx_pci_pm_resume(struct device * dev)710 static int t7xx_pci_pm_resume(struct device *dev)
711 {
712 return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
713 }
714
t7xx_pci_pm_thaw(struct device * dev)715 static int t7xx_pci_pm_thaw(struct device *dev)
716 {
717 return __t7xx_pci_pm_resume(to_pci_dev(dev), false);
718 }
719
t7xx_pci_pm_runtime_suspend(struct device * dev)720 static int t7xx_pci_pm_runtime_suspend(struct device *dev)
721 {
722 return __t7xx_pci_pm_suspend(to_pci_dev(dev));
723 }
724
t7xx_pci_pm_runtime_resume(struct device * dev)725 static int t7xx_pci_pm_runtime_resume(struct device *dev)
726 {
727 return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
728 }
729
730 static const struct dev_pm_ops t7xx_pci_pm_ops = {
731 .prepare = t7xx_pci_pm_prepare,
732 .suspend = t7xx_pci_pm_suspend,
733 .resume = t7xx_pci_pm_resume,
734 .resume_noirq = t7xx_pci_pm_resume_noirq,
735 .freeze = t7xx_pci_pm_suspend,
736 .thaw = t7xx_pci_pm_thaw,
737 .poweroff = t7xx_pci_pm_suspend,
738 .restore = t7xx_pci_pm_resume,
739 .restore_noirq = t7xx_pci_pm_resume_noirq,
740 .runtime_suspend = t7xx_pci_pm_runtime_suspend,
741 .runtime_resume = t7xx_pci_pm_runtime_resume
742 };
743
t7xx_request_irq(struct pci_dev * pdev)744 static int t7xx_request_irq(struct pci_dev *pdev)
745 {
746 struct t7xx_pci_dev *t7xx_dev;
747 int ret = 0, i;
748
749 t7xx_dev = pci_get_drvdata(pdev);
750
751 for (i = 0; i < EXT_INT_NUM; i++) {
752 const char *irq_descr;
753 int irq_vec;
754
755 if (!t7xx_dev->intr_handler[i])
756 continue;
757
758 irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
759 dev_driver_string(&pdev->dev), i);
760 if (!irq_descr) {
761 ret = -ENOMEM;
762 break;
763 }
764
765 irq_vec = pci_irq_vector(pdev, i);
766 ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i],
767 t7xx_dev->intr_thread[i], 0, irq_descr,
768 t7xx_dev->callback_param[i]);
769 if (ret) {
770 dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret);
771 break;
772 }
773 }
774
775 if (ret) {
776 while (i--) {
777 if (!t7xx_dev->intr_handler[i])
778 continue;
779
780 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
781 }
782 }
783
784 return ret;
785 }
786
t7xx_setup_msix(struct t7xx_pci_dev * t7xx_dev)787 static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev)
788 {
789 struct pci_dev *pdev = t7xx_dev->pdev;
790 int ret;
791
792 /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */
793 ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX);
794 if (ret < 0) {
795 dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret);
796 return ret;
797 }
798
799 ret = t7xx_request_irq(pdev);
800 if (ret) {
801 pci_free_irq_vectors(pdev);
802 return ret;
803 }
804
805 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
806 return 0;
807 }
808
t7xx_interrupt_init(struct t7xx_pci_dev * t7xx_dev)809 static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev)
810 {
811 int ret, i;
812
813 if (!t7xx_dev->pdev->msix_cap)
814 return -EINVAL;
815
816 ret = t7xx_setup_msix(t7xx_dev);
817 if (ret)
818 return ret;
819
820 /* IPs enable interrupts when ready */
821 for (i = 0; i < EXT_INT_NUM; i++)
822 t7xx_pcie_mac_set_int(t7xx_dev, i);
823
824 return 0;
825 }
826
t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev * t7xx_dev)827 static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev)
828 {
829 t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base +
830 INFRACFG_AO_DEV_CHIP -
831 t7xx_dev->base_addr.pcie_dev_reg_trsl_addr;
832 }
833
t7xx_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)834 static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
835 {
836 struct t7xx_pci_dev *t7xx_dev;
837 void __iomem *iomem;
838 int ret;
839
840 t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL);
841 if (!t7xx_dev)
842 return -ENOMEM;
843
844 pci_set_drvdata(pdev, t7xx_dev);
845 t7xx_dev->pdev = pdev;
846
847 ret = pcim_enable_device(pdev);
848 if (ret)
849 return ret;
850
851 pci_set_master(pdev);
852
853 iomem = pcim_iomap_region(pdev, T7XX_PCI_IREG_BASE, DRIVER_NAME);
854 ret = PTR_ERR_OR_ZERO(iomem);
855 if (ret) {
856 dev_err(&pdev->dev, "Could not request IREG BAR: %d\n", ret);
857 return -ENOMEM;
858 }
859 IREG_BASE(t7xx_dev) = iomem;
860
861 iomem = pcim_iomap_region(pdev, T7XX_PCI_EREG_BASE, DRIVER_NAME);
862 ret = PTR_ERR_OR_ZERO(iomem);
863 if (ret) {
864 dev_err(&pdev->dev, "Could not request EREG BAR: %d\n", ret);
865 return -ENOMEM;
866 }
867 t7xx_dev->base_addr.pcie_ext_reg_base = iomem;
868
869 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
870 if (ret) {
871 dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret);
872 return ret;
873 }
874
875 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
876 if (ret) {
877 dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret);
878 return ret;
879 }
880
881 ret = t7xx_pci_pm_init(t7xx_dev);
882 if (ret)
883 return ret;
884
885 t7xx_pcie_mac_atr_init(t7xx_dev);
886 t7xx_pci_infracfg_ao_calc(t7xx_dev);
887 t7xx_mhccif_init(t7xx_dev);
888
889 ret = t7xx_md_init(t7xx_dev);
890 if (ret)
891 return ret;
892
893 t7xx_pcie_mac_interrupts_dis(t7xx_dev);
894
895 ret = sysfs_create_group(&t7xx_dev->pdev->dev.kobj,
896 &t7xx_attribute_group);
897 if (ret)
898 goto err_md_exit;
899
900 ret = t7xx_interrupt_init(t7xx_dev);
901 if (ret)
902 goto err_remove_group;
903
904
905 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
906 t7xx_pcie_mac_interrupts_en(t7xx_dev);
907
908 return 0;
909
910 err_remove_group:
911 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj,
912 &t7xx_attribute_group);
913
914 err_md_exit:
915 t7xx_md_exit(t7xx_dev);
916 return ret;
917 }
918
t7xx_pci_remove(struct pci_dev * pdev)919 static void t7xx_pci_remove(struct pci_dev *pdev)
920 {
921 struct t7xx_pci_dev *t7xx_dev;
922 int i;
923
924 t7xx_dev = pci_get_drvdata(pdev);
925
926 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj,
927 &t7xx_attribute_group);
928 t7xx_md_exit(t7xx_dev);
929
930 for (i = 0; i < EXT_INT_NUM; i++) {
931 if (!t7xx_dev->intr_handler[i])
932 continue;
933
934 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
935 }
936
937 pci_free_irq_vectors(t7xx_dev->pdev);
938 }
939
940 static const struct pci_device_id t7xx_pci_table[] = {
941 { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) },
942 { PCI_DEVICE(0x14c0, 0x4d75) }, // Dell DW5933e
943 { }
944 };
945 MODULE_DEVICE_TABLE(pci, t7xx_pci_table);
946
947 static struct pci_driver t7xx_pci_driver = {
948 .name = DRIVER_NAME,
949 .id_table = t7xx_pci_table,
950 .probe = t7xx_pci_probe,
951 .remove = t7xx_pci_remove,
952 .driver.pm = &t7xx_pci_pm_ops,
953 .shutdown = t7xx_pci_shutdown,
954 };
955
956 module_pci_driver(t7xx_pci_driver);
957
958 MODULE_AUTHOR("MediaTek Inc");
959 MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver");
960 MODULE_LICENSE("GPL");
961