1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /* Copyright (C) 2023 MediaTek Inc. */
3
4 #include <linux/module.h>
5 #include <linux/firmware.h>
6
7 #include "mt792x.h"
8 #include "dma.h"
9 #include "trace.h"
10
mt792x_irq_handler(int irq,void * dev_instance)11 irqreturn_t mt792x_irq_handler(int irq, void *dev_instance)
12 {
13 struct mt792x_dev *dev = dev_instance;
14
15 if (test_bit(MT76_REMOVED, &dev->mt76.phy.state))
16 return IRQ_NONE;
17 mt76_wr(dev, dev->irq_map->host_irq_enable, 0);
18
19 if (!test_bit(MT76_STATE_INITIALIZED, &dev->mphy.state))
20 return IRQ_NONE;
21
22 tasklet_schedule(&dev->mt76.irq_tasklet);
23
24 return IRQ_HANDLED;
25 }
26 EXPORT_SYMBOL_GPL(mt792x_irq_handler);
27
mt792x_irq_tasklet(unsigned long data)28 void mt792x_irq_tasklet(unsigned long data)
29 {
30 struct mt792x_dev *dev = (struct mt792x_dev *)data;
31 const struct mt792x_irq_map *irq_map = dev->irq_map;
32 u32 intr, mask = 0;
33
34 mt76_wr(dev, irq_map->host_irq_enable, 0);
35
36 intr = mt76_rr(dev, MT_WFDMA0_HOST_INT_STA);
37 intr &= dev->mt76.mmio.irqmask;
38 mt76_wr(dev, MT_WFDMA0_HOST_INT_STA, intr);
39
40 trace_dev_irq(&dev->mt76, intr, dev->mt76.mmio.irqmask);
41
42 mask |= intr & irq_map->rx.all_complete_mask;
43 if (intr & dev->irq_map->tx.mcu_complete_mask)
44 mask |= dev->irq_map->tx.mcu_complete_mask;
45
46 if (intr & MT_INT_MCU_CMD) {
47 u32 intr_sw;
48
49 intr_sw = mt76_rr(dev, MT_MCU_CMD);
50 /* ack MCU2HOST_SW_INT_STA */
51 mt76_wr(dev, MT_MCU_CMD, intr_sw);
52 if (intr_sw & MT_MCU_CMD_WAKE_RX_PCIE) {
53 mask |= irq_map->rx.data_complete_mask;
54 intr |= irq_map->rx.data_complete_mask;
55 }
56 }
57
58 mt76_set_irq_mask(&dev->mt76, irq_map->host_irq_enable, mask, 0);
59
60 if (intr & dev->irq_map->tx.all_complete_mask)
61 napi_schedule(&dev->mt76.tx_napi);
62
63 if (intr & irq_map->rx.wm_complete_mask)
64 napi_schedule(&dev->mt76.napi[MT_RXQ_MCU]);
65
66 if (intr & irq_map->rx.wm2_complete_mask)
67 napi_schedule(&dev->mt76.napi[MT_RXQ_MCU_WA]);
68
69 if (intr & irq_map->rx.data_complete_mask)
70 napi_schedule(&dev->mt76.napi[MT_RXQ_MAIN]);
71 }
72 EXPORT_SYMBOL_GPL(mt792x_irq_tasklet);
73
mt792x_rx_poll_complete(struct mt76_dev * mdev,enum mt76_rxq_id q)74 void mt792x_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q)
75 {
76 struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76);
77 const struct mt792x_irq_map *irq_map = dev->irq_map;
78
79 if (q == MT_RXQ_MAIN)
80 mt76_connac_irq_enable(mdev, irq_map->rx.data_complete_mask);
81 else if (q == MT_RXQ_MCU_WA)
82 mt76_connac_irq_enable(mdev, irq_map->rx.wm2_complete_mask);
83 else
84 mt76_connac_irq_enable(mdev, irq_map->rx.wm_complete_mask);
85 }
86 EXPORT_SYMBOL_GPL(mt792x_rx_poll_complete);
87
mt792x_dma_alloc_queues(struct mt792x_dev * dev,const struct mt792x_dma_layout * layout)88 int mt792x_dma_alloc_queues(struct mt792x_dev *dev,
89 const struct mt792x_dma_layout *layout)
90 {
91 int ret;
92
93 mt76_dma_attach(&dev->mt76);
94
95 ret = mt792x_dma_disable(dev, true);
96 if (ret)
97 return ret;
98
99 /* init tx queue */
100 ret = mt76_connac_init_tx_queues(dev->phy.mt76, layout->tx_data0.qid,
101 layout->tx_data0.n_desc,
102 layout->tx_data0.ring_base,
103 NULL, 0);
104 if (ret)
105 return ret;
106
107 mt76_wr(dev, MT_WFDMA0_TX_RING0_EXT_CTRL, 0x4);
108
109 /* command to WM */
110 ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_WM,
111 layout->tx_mcu.qid,
112 layout->tx_mcu.n_desc,
113 layout->tx_mcu.ring_base);
114 if (ret)
115 return ret;
116
117 /* firmware download */
118 ret = mt76_init_mcu_queue(&dev->mt76, MT_MCUQ_FWDL,
119 layout->tx_fwdl.qid,
120 layout->tx_fwdl.n_desc,
121 layout->tx_fwdl.ring_base);
122 if (ret)
123 return ret;
124
125 /* tx done */
126 if (layout->tx_done.ring_base) {
127 ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU_WA],
128 layout->tx_done.qid,
129 layout->tx_done.n_desc,
130 MT_RX_BUF_SIZE,
131 layout->tx_done.ring_base);
132 if (ret)
133 return ret;
134 }
135
136 /* rx event */
137 ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MCU],
138 layout->rx_mcu.qid,
139 layout->rx_mcu.n_desc,
140 MT_RX_BUF_SIZE,
141 layout->rx_mcu.ring_base);
142 if (ret)
143 return ret;
144
145 /* rx data */
146 ret = mt76_queue_alloc(dev, &dev->mt76.q_rx[MT_RXQ_MAIN],
147 layout->rx_data.qid,
148 layout->rx_data.n_desc,
149 MT_RX_BUF_SIZE,
150 layout->rx_data.ring_base);
151 return ret;
152 }
153 EXPORT_SYMBOL_GPL(mt792x_dma_alloc_queues);
154
155 #define PREFETCH(base, depth) ((base) << 16 | (depth))
156
mt7925_dma_prefetch_setup(struct mt792x_dev * dev)157 static void mt7925_dma_prefetch_setup(struct mt792x_dev *dev)
158 {
159 /* rx ring */
160 mt76_wr(dev, MT_WFDMA0_RX_RING0_EXT_CTRL, PREFETCH(0x0000, 0x4));
161 mt76_wr(dev, MT_WFDMA0_RX_RING1_EXT_CTRL, PREFETCH(0x0040, 0x4));
162 mt76_wr(dev, MT_WFDMA0_RX_RING2_EXT_CTRL, PREFETCH(0x0080, 0x4));
163 mt76_wr(dev, MT_WFDMA0_RX_RING3_EXT_CTRL, PREFETCH(0x00c0, 0x4));
164
165 /* tx ring */
166 mt76_wr(dev, MT_WFDMA0_TX_RING0_EXT_CTRL, PREFETCH(0x0100, 0x10));
167 mt76_wr(dev, MT_WFDMA0_TX_RING1_EXT_CTRL, PREFETCH(0x0200, 0x10));
168 mt76_wr(dev, MT_WFDMA0_TX_RING2_EXT_CTRL, PREFETCH(0x0300, 0x10));
169 mt76_wr(dev, MT_WFDMA0_TX_RING3_EXT_CTRL, PREFETCH(0x0400, 0x10));
170 mt76_wr(dev, MT_WFDMA0_TX_RING15_EXT_CTRL, PREFETCH(0x0500, 0x4));
171 mt76_wr(dev, MT_WFDMA0_TX_RING16_EXT_CTRL, PREFETCH(0x0540, 0x4));
172 }
173
mt7925_wfdma_setup(struct mt792x_dev * dev)174 static void mt7925_wfdma_setup(struct mt792x_dev *dev)
175 {
176 mt76_rmw(dev, MT_UWFDMA0_GLO_CFG_EXT1, BIT(28), BIT(28));
177 mt76_set(dev, MT_WFDMA0_INT_RX_PRI, 0x0F00);
178 mt76_set(dev, MT_WFDMA0_INT_TX_PRI, 0x7F00);
179 }
180
mt7927_dma_prefetch_setup(struct mt792x_dev * dev)181 static void mt7927_dma_prefetch_setup(struct mt792x_dev *dev)
182 {
183 mt76_wr(dev, MT_WFDMA_PREFETCH_CTRL,
184 mt76_rr(dev, MT_WFDMA_PREFETCH_CTRL));
185 mt76_wr(dev, MT_WFDMA_PREFETCH_CFG0, 0x660077);
186 mt76_wr(dev, MT_WFDMA_PREFETCH_CFG1, 0x1100);
187 mt76_wr(dev, MT_WFDMA_PREFETCH_CFG2, 0x30004f);
188 mt76_wr(dev, MT_WFDMA_PREFETCH_CFG3, 0x542200);
189 mt76_wr(dev, MT_WFDMA0_RX_RING4_EXT_CTRL, PREFETCH(0x0000, 0x8));
190 mt76_wr(dev, MT_WFDMA0_RX_RING6_EXT_CTRL, PREFETCH(0x0080, 0x8));
191 mt76_wr(dev, MT_WFDMA0_RX_RING7_EXT_CTRL, PREFETCH(0x0100, 0x4));
192 mt76_wr(dev, MT_WFDMA0_TX_RING16_EXT_CTRL, PREFETCH(0x0140, 0x4));
193 mt76_wr(dev, MT_WFDMA0_TX_RING15_EXT_CTRL, PREFETCH(0x0180, 0x10));
194 mt76_wr(dev, MT_WFDMA0_TX_RING0_EXT_CTRL, PREFETCH(0x0280, 0x4));
195 }
196
mt7927_wfdma_setup(struct mt792x_dev * dev)197 static void mt7927_wfdma_setup(struct mt792x_dev *dev)
198 {
199 mt76_set(dev, MT_WFDMA0_GLO_CFG,
200 MT_WFDMA0_GLO_CFG_ADDR_EXT_EN |
201 MT_WFDMA0_GLO_CFG_FW_DWLD_BYPASS_DMASHDL);
202 mt76_clear(dev, MT_WFDMA0_GLO_CFG,
203 MT_WFDMA0_GLO_CFG_CSR_LBK_RX_Q_SEL_EN);
204 mt76_rmw(dev, MT_WFDMA0_GLO_CFG_EXT1, BIT(28), BIT(28));
205 mt76_set(dev, MT_WFDMA0_INT_RX_PRI, 0x0F00);
206 mt76_set(dev, MT_WFDMA0_INT_TX_PRI, 0x7F00);
207 }
208
mt7928_dma_prefetch_setup(struct mt792x_dev * dev)209 static void mt7928_dma_prefetch_setup(struct mt792x_dev *dev)
210 {
211 /* rx ring */
212 mt76_wr(dev, MT_WFDMA0_RX_RING0_EXT_CTRL, PREFETCH(0x0000, 0x8));
213 mt76_wr(dev, MT_WFDMA0_RX_RING1_EXT_CTRL, PREFETCH(0x0080, 0x4));
214 mt76_wr(dev, MT_WFDMA0_RX_RING2_EXT_CTRL, PREFETCH(0x00c0, 0x8));
215 mt76_wr(dev, MT_WFDMA0_RX_RING3_EXT_CTRL, PREFETCH(0x0140, 0x4));
216 mt76_wr(dev, MT_WFDMA0_RX_RING4_EXT_CTRL, PREFETCH(0x0180, 0x4));
217 mt76_wr(dev, MT_WFDMA0_RX_RING5_EXT_CTRL, PREFETCH(0x01c0, 0x4));
218 mt76_wr(dev, MT_WFDMA0_RX_RING6_EXT_CTRL, PREFETCH(0x0200, 0x4));
219 /* tx ring */
220 mt76_wr(dev, MT_WFDMA0_TX_RING0_EXT_CTRL, PREFETCH(0x0240, 0x10));
221 mt76_wr(dev, MT_WFDMA0_TX_RING1_EXT_CTRL, PREFETCH(0x0340, 0x10));
222 mt76_wr(dev, MT_WFDMA0_TX_RING2_EXT_CTRL, PREFETCH(0x0440, 0x10));
223 mt76_wr(dev, MT_WFDMA0_TX_RING3_EXT_CTRL, PREFETCH(0x0540, 0x10));
224 mt76_wr(dev, MT_WFDMA0_TX_RING4_EXT_CTRL, PREFETCH(0x0640, 0x10));
225 mt76_wr(dev, MT_WFDMA0_TX_RING5_EXT_CTRL, PREFETCH(0x0740, 0x10));
226 mt76_wr(dev, MT_WFDMA0_TX_RING15_EXT_CTRL, PREFETCH(0x0840, 0x4));
227 mt76_wr(dev, MT_WFDMA0_TX_RING16_EXT_CTRL, PREFETCH(0x0880, 0x4));
228 }
229
mt7928_wfdma_setup(struct mt792x_dev * dev)230 static void mt7928_wfdma_setup(struct mt792x_dev *dev)
231 {
232 mt76_wr(dev, MT_WFDMA0_INT_RX_PRI, 0);
233 mt76_wr(dev, MT_WFDMA0_INT_TX_PRI, 0);
234 }
235
mt792x_dma_prefetch(struct mt792x_dev * dev)236 static void mt792x_dma_prefetch(struct mt792x_dev *dev)
237 {
238 if (is_mt7928(&dev->mt76)) {
239 mt7928_dma_prefetch_setup(dev);
240 } else if (is_mt7927(&dev->mt76)) {
241 mt7927_dma_prefetch_setup(dev);
242 } else if (is_mt7925(&dev->mt76)) {
243 mt7925_dma_prefetch_setup(dev);
244 } else if (is_mt7902(&dev->mt76)) {
245 /* rx ring */
246 mt76_wr(dev, MT_WFDMA0_RX_RING0_EXT_CTRL, PREFETCH(0x0000, 0x4));
247 mt76_wr(dev, MT_WFDMA0_RX_RING1_EXT_CTRL, PREFETCH(0x0040, 0x4));
248 mt76_wr(dev, MT_WFDMA0_RX_RING2_EXT_CTRL, PREFETCH(0x0080, 0x4));
249 mt76_wr(dev, MT_WFDMA0_RX_RING3_EXT_CTRL, PREFETCH(0x00c0, 0x4));
250 /* tx ring */
251 mt76_wr(dev, MT_WFDMA0_TX_RING0_EXT_CTRL, PREFETCH(0x0100, 0x4));
252 mt76_wr(dev, MT_WFDMA0_TX_RING1_EXT_CTRL, PREFETCH(0x0140, 0x4));
253 mt76_wr(dev, MT_WFDMA0_TX_RING2_EXT_CTRL, PREFETCH(0x0180, 0x4));
254 mt76_wr(dev, MT_WFDMA0_TX_RING3_EXT_CTRL, PREFETCH(0x01c0, 0x4));
255 mt76_wr(dev, MT_WFDMA0_TX_RING4_EXT_CTRL, PREFETCH(0x0200, 0x4));
256 mt76_wr(dev, MT_WFDMA0_TX_RING5_EXT_CTRL, PREFETCH(0x0240, 0x4));
257 mt76_wr(dev, MT_WFDMA0_TX_RING6_EXT_CTRL, PREFETCH(0x0280, 0x4));
258 mt76_wr(dev, MT_WFDMA0_TX_RING15_EXT_CTRL, PREFETCH(0x02c0, 0x4));
259 mt76_wr(dev, MT_WFDMA0_TX_RING16_EXT_CTRL, PREFETCH(0x0300, 0x4));
260 } else {
261 /* rx ring */
262 mt76_wr(dev, MT_WFDMA0_RX_RING0_EXT_CTRL, PREFETCH(0x0, 0x4));
263 mt76_wr(dev, MT_WFDMA0_RX_RING2_EXT_CTRL, PREFETCH(0x40, 0x4));
264 mt76_wr(dev, MT_WFDMA0_RX_RING3_EXT_CTRL, PREFETCH(0x80, 0x4));
265 mt76_wr(dev, MT_WFDMA0_RX_RING4_EXT_CTRL, PREFETCH(0xc0, 0x4));
266 mt76_wr(dev, MT_WFDMA0_RX_RING5_EXT_CTRL, PREFETCH(0x100, 0x4));
267 /* tx ring */
268 mt76_wr(dev, MT_WFDMA0_TX_RING0_EXT_CTRL, PREFETCH(0x140, 0x4));
269 mt76_wr(dev, MT_WFDMA0_TX_RING1_EXT_CTRL, PREFETCH(0x180, 0x4));
270 mt76_wr(dev, MT_WFDMA0_TX_RING2_EXT_CTRL, PREFETCH(0x1c0, 0x4));
271 mt76_wr(dev, MT_WFDMA0_TX_RING3_EXT_CTRL, PREFETCH(0x200, 0x4));
272 mt76_wr(dev, MT_WFDMA0_TX_RING4_EXT_CTRL, PREFETCH(0x240, 0x4));
273 mt76_wr(dev, MT_WFDMA0_TX_RING5_EXT_CTRL, PREFETCH(0x280, 0x4));
274 mt76_wr(dev, MT_WFDMA0_TX_RING6_EXT_CTRL, PREFETCH(0x2c0, 0x4));
275 mt76_wr(dev, MT_WFDMA0_TX_RING16_EXT_CTRL, PREFETCH(0x340, 0x4));
276 mt76_wr(dev, MT_WFDMA0_TX_RING17_EXT_CTRL, PREFETCH(0x380, 0x4));
277 }
278 }
279
mt792x_dma_enable(struct mt792x_dev * dev)280 int mt792x_dma_enable(struct mt792x_dev *dev)
281 {
282 u32 addr;
283
284 /* configure prefetch settings */
285 mt792x_dma_prefetch(dev);
286
287 /* reset dma idx */
288 mt76_wr(dev, MT_WFDMA0_RST_DTX_PTR, ~0);
289 if (is_mt7925(&dev->mt76) || is_mt7928(&dev->mt76))
290 mt76_wr(dev, MT_WFDMA0_RST_DRX_PTR, ~0);
291
292 /* configure delay interrupt */
293 mt76_wr(dev, MT_WFDMA0_PRI_DLY_INT_CFG0, 0);
294
295 if (is_mt7928(&dev->mt76)) {
296 mt76_wr(dev, MT_WFDMA0_GLO_CFG,
297 MT_WFDMA0_GLO_CFG_TX_WB_DDONE |
298 MT_WFDMA0_GLO_CFG_FIFO_LITTLE_ENDIAN |
299 MT_WFDMA0_GLO_CFG_CLK_GAT_DIS |
300 MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
301 FIELD_PREP(MT_WFDMA0_GLO_CFG_DMA_SIZE, 1) |
302 MT_WFDMA0_GLO_CFG_FIFO_DIS_CHECK |
303 MT_WFDMA0_GLO_CFG_RX_WB_DDONE |
304 MT_WFDMA0_GLO_CFG_CSR_DISP_BASE_PTR_CHAIN_EN |
305 MT_WFDMA0_GLO_CFG_ADDR_EXT_EN |
306 MT_WFDMA0_GLO_CFG_CSR_LBK_RX_Q_SEL_EN);
307 /* set rxq threshold to 2 */
308 for (addr = MT_WFDMA0_WPDMA_PAUSE_RXQ_TH10;
309 addr <= MT_WFDMA0_WPDMA_PAUSE_RXQ_TH76;
310 addr += 4) {
311 mt76_wr(dev, addr,
312 FIELD_PREP(MT_WFDMA0_WPDMA_PAUSE_RXQ_THXX_L_TH_MASK, 2) |
313 FIELD_PREP(MT_WFDMA0_WPDMA_PAUSE_RXQ_THXX_H_TH_MASK, 2));
314 }
315 mt76_wr(dev, MT_WFDMA0_GLO_CFG_EXT0,
316 MT_WFDMA0_GLO_CFG_EXT0_CSR_MEM_ARB_LOCK_EN |
317 MT_WFDMA0_GLO_CFG_EXT0_CSR_TX_DMASHDL_LITE_EN |
318 MT_WFDMA0_GLO_CFG_EXT0_CSR_RX_WB_KEEP_RSVD |
319 MT_WFDMA0_GLO_CFG_EXT0_CSR_BID_CHECK_BYPASS_EN |
320 MT_WFDMA0_GLO_CFG_EXT0_CSR_RX_INFO_WB_EN |
321 MT_WFDMA0_GLO_CFG_EXT0_CSR_AXI_AWUSER_LOCK_EN |
322 FIELD_PREP(MT_WFDMA0_GLO_CFG_EXT0_CSR_MAX_PREFETCH_CNT_MASK, 3) |
323 FIELD_PREP(MT_WFDMA0_GLO_CFG_EXT0_CSR_MEM_BST_SIZE_MASK, 3) |
324 FIELD_PREP(MT_WFDMA0_GLO_CFG_EXT0_CSR_AXI_AW_OUTSTANDING_NUM_MASK,
325 8));
326 } else {
327 mt76_set(dev, MT_WFDMA0_GLO_CFG,
328 MT_WFDMA0_GLO_CFG_TX_WB_DDONE |
329 MT_WFDMA0_GLO_CFG_FIFO_LITTLE_ENDIAN |
330 MT_WFDMA0_GLO_CFG_CLK_GAT_DIS |
331 MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
332 FIELD_PREP(MT_WFDMA0_GLO_CFG_DMA_SIZE, 3) |
333 MT_WFDMA0_GLO_CFG_FIFO_DIS_CHECK |
334 MT_WFDMA0_GLO_CFG_RX_WB_DDONE |
335 MT_WFDMA0_GLO_CFG_CSR_DISP_BASE_PTR_CHAIN_EN |
336 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
337 }
338
339 mt76_set(dev, MT_WFDMA0_GLO_CFG,
340 MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN);
341
342 if (is_mt7928(&dev->mt76))
343 mt7928_wfdma_setup(dev);
344 else if (is_mt7927(&dev->mt76))
345 mt7927_wfdma_setup(dev);
346 else if (is_mt7925(&dev->mt76))
347 mt7925_wfdma_setup(dev);
348
349 mt76_set(dev, MT_WFDMA_DUMMY_CR, MT_WFDMA_NEED_REINIT);
350
351 /* enable interrupts for TX/RX rings */
352 mt76_connac_irq_enable(&dev->mt76,
353 dev->irq_map->tx.all_complete_mask |
354 dev->irq_map->rx.all_complete_mask |
355 MT_INT_MCU_CMD);
356 mt76_set(dev, MT_MCU2HOST_SW_INT_ENA, MT_MCU_CMD_WAKE_RX_PCIE);
357
358 return 0;
359 }
360 EXPORT_SYMBOL_GPL(mt792x_dma_enable);
361
362 static int
mt792x_dma_reset(struct mt792x_dev * dev,bool force)363 mt792x_dma_reset(struct mt792x_dev *dev, bool force)
364 {
365 int i, err;
366
367 err = mt792x_dma_disable(dev, force);
368 if (err)
369 return err;
370
371 /* reset hw queues */
372 for (i = 0; i < __MT_TXQ_MAX; i++)
373 mt76_queue_reset(dev, dev->mphy.q_tx[i], true);
374
375 for (i = 0; i < __MT_MCUQ_MAX; i++)
376 mt76_queue_reset(dev, dev->mt76.q_mcu[i], true);
377
378 mt76_for_each_q_rx(&dev->mt76, i)
379 mt76_queue_reset(dev, &dev->mt76.q_rx[i], true);
380
381 mt76_tx_status_check(&dev->mt76, true);
382
383 return mt792x_dma_enable(dev);
384 }
385
mt792x_wpdma_reset(struct mt792x_dev * dev,bool force)386 int mt792x_wpdma_reset(struct mt792x_dev *dev, bool force)
387 {
388 int i, err;
389
390 /* clean up hw queues */
391 for (i = 0; i < ARRAY_SIZE(dev->mt76.phy.q_tx); i++)
392 mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[i], true);
393
394 for (i = 0; i < ARRAY_SIZE(dev->mt76.q_mcu); i++)
395 mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[i], true);
396
397 mt76_for_each_q_rx(&dev->mt76, i)
398 mt76_queue_rx_cleanup(dev, &dev->mt76.q_rx[i]);
399
400 if (force) {
401 err = mt792x_wfsys_reset(dev);
402 if (err)
403 return err;
404 }
405 err = mt792x_dma_reset(dev, force);
406 if (err)
407 return err;
408
409 mt76_for_each_q_rx(&dev->mt76, i)
410 mt76_queue_rx_reset(dev, i);
411
412 return 0;
413 }
414 EXPORT_SYMBOL_GPL(mt792x_wpdma_reset);
415
mt792x_wpdma_reinit_cond(struct mt792x_dev * dev)416 int mt792x_wpdma_reinit_cond(struct mt792x_dev *dev)
417 {
418 struct mt76_connac_pm *pm = &dev->pm;
419 int err;
420
421 if (dev->skip_wpdma_reinit)
422 return 0;
423
424 /* check if the wpdma must be reinitialized */
425 if (mt792x_dma_need_reinit(dev)) {
426 /* disable interrutpts */
427 mt76_wr(dev, dev->irq_map->host_irq_enable, 0);
428 if (dev->pcie_reg)
429 mt76_wr(dev, dev->pcie_reg->imask, 0x0);
430
431 err = mt792x_wpdma_reset(dev, false);
432 if (err) {
433 dev_err(dev->mt76.dev, "wpdma reset failed\n");
434 return err;
435 }
436
437 /* enable interrutpts */
438 if (dev->pcie_reg)
439 mt76_wr(dev, dev->pcie_reg->imask, 0xff);
440 pm->stats.lp_wake++;
441 }
442
443 return 0;
444 }
445 EXPORT_SYMBOL_GPL(mt792x_wpdma_reinit_cond);
446
mt792x_dma_disable(struct mt792x_dev * dev,bool force)447 int mt792x_dma_disable(struct mt792x_dev *dev, bool force)
448 {
449 /* disable WFDMA0 */
450 mt76_clear(dev, MT_WFDMA0_GLO_CFG,
451 MT_WFDMA0_GLO_CFG_TX_DMA_EN | MT_WFDMA0_GLO_CFG_RX_DMA_EN |
452 MT_WFDMA0_GLO_CFG_CSR_DISP_BASE_PTR_CHAIN_EN |
453 MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
454 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
455 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
456
457 if (!mt76_poll_msec_tick(dev, MT_WFDMA0_GLO_CFG,
458 MT_WFDMA0_GLO_CFG_TX_DMA_BUSY |
459 MT_WFDMA0_GLO_CFG_RX_DMA_BUSY, 0, 100, 1))
460 return -ETIMEDOUT;
461
462 if (is_mt7927(&dev->mt76)) {
463 mt76_wr(dev, MT_WFDMA0_RST_DTX_PTR, ~0);
464 mt76_wr(dev, MT_WFDMA0_RST_DRX_PTR, ~0);
465 }
466
467 /* disable dmashdl */
468 mt76_clear(dev, MT_WFDMA0_GLO_CFG_EXT0,
469 MT_WFDMA0_GLO_CFG_EXT0_CSR_TX_DMASHDL_EN);
470 mt76_set(dev, MT_DMASHDL_SW_CONTROL, MT_DMASHDL_DMASHDL_BYPASS);
471
472 if (force) {
473 /* reset */
474 mt76_clear(dev, MT_WFDMA0_RST,
475 MT_WFDMA0_RST_DMASHDL_ALL_RST |
476 MT_WFDMA0_RST_LOGIC_RST);
477
478 mt76_set(dev, MT_WFDMA0_RST,
479 MT_WFDMA0_RST_DMASHDL_ALL_RST |
480 MT_WFDMA0_RST_LOGIC_RST);
481 }
482
483 return 0;
484 }
485 EXPORT_SYMBOL_GPL(mt792x_dma_disable);
486
mt792x_dma_cleanup(struct mt792x_dev * dev)487 void mt792x_dma_cleanup(struct mt792x_dev *dev)
488 {
489 /* disable */
490 mt76_clear(dev, MT_WFDMA0_GLO_CFG,
491 MT_WFDMA0_GLO_CFG_TX_DMA_EN |
492 MT_WFDMA0_GLO_CFG_RX_DMA_EN |
493 MT_WFDMA0_GLO_CFG_CSR_DISP_BASE_PTR_CHAIN_EN |
494 MT_WFDMA0_GLO_CFG_OMIT_TX_INFO |
495 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO |
496 MT_WFDMA0_GLO_CFG_OMIT_RX_INFO_PFET2);
497
498 mt76_poll_msec_tick(dev, MT_WFDMA0_GLO_CFG,
499 MT_WFDMA0_GLO_CFG_TX_DMA_BUSY |
500 MT_WFDMA0_GLO_CFG_RX_DMA_BUSY, 0, 100, 1);
501
502 /* reset */
503 mt76_clear(dev, MT_WFDMA0_RST,
504 MT_WFDMA0_RST_DMASHDL_ALL_RST |
505 MT_WFDMA0_RST_LOGIC_RST);
506
507 mt76_set(dev, MT_WFDMA0_RST,
508 MT_WFDMA0_RST_DMASHDL_ALL_RST |
509 MT_WFDMA0_RST_LOGIC_RST);
510
511 mt76_dma_cleanup(&dev->mt76);
512 }
513 EXPORT_SYMBOL_GPL(mt792x_dma_cleanup);
514
mt792x_poll_tx(struct napi_struct * napi,int budget)515 int mt792x_poll_tx(struct napi_struct *napi, int budget)
516 {
517 struct mt792x_dev *dev;
518
519 dev = container_of(napi, struct mt792x_dev, mt76.tx_napi);
520
521 if (!mt76_connac_pm_ref(&dev->mphy, &dev->pm)) {
522 napi_complete(napi);
523 queue_work(dev->mt76.wq, &dev->pm.wake_work);
524 return 0;
525 }
526
527 mt76_connac_tx_cleanup(&dev->mt76);
528 if (napi_complete(napi))
529 mt76_connac_irq_enable(&dev->mt76,
530 dev->irq_map->tx.all_complete_mask);
531 mt76_connac_pm_unref(&dev->mphy, &dev->pm);
532
533 return 0;
534 }
535 EXPORT_SYMBOL_GPL(mt792x_poll_tx);
536
mt792x_poll_rx(struct napi_struct * napi,int budget)537 int mt792x_poll_rx(struct napi_struct *napi, int budget)
538 {
539 struct mt792x_dev *dev;
540 int done;
541
542 dev = mt76_priv(napi->dev);
543
544 if (!mt76_connac_pm_ref(&dev->mphy, &dev->pm)) {
545 napi_complete(napi);
546 queue_work(dev->mt76.wq, &dev->pm.wake_work);
547 return 0;
548 }
549 done = mt76_dma_rx_poll(napi, budget);
550 mt76_connac_pm_unref(&dev->mphy, &dev->pm);
551
552 return done;
553 }
554 EXPORT_SYMBOL_GPL(mt792x_poll_rx);
555
mt7927_sema_status_read(struct mt792x_dev * dev,u32 addr)556 static void mt7927_sema_status_read(struct mt792x_dev *dev, u32 addr)
557 {
558 u32 remap;
559
560 remap = mt76_rr(dev, MT7927_PCIE2AP_REMAP_WF_0_54);
561 mt76_wr(dev, MT7927_PCIE2AP_REMAP_WF_0_54,
562 (remap & ~MT7927_PCIE2AP_REMAP_WF_0_54_MASK) |
563 MT7927_PCIE2AP_REMAP_WF_0_54_VAL);
564 usleep_range(10, 20);
565
566 mt76_rr(dev, addr);
567
568 mt76_wr(dev, MT7927_PCIE2AP_REMAP_WF_0_54, remap);
569 usleep_range(10, 20);
570 }
571
mt7927_wfsys_reset(struct mt792x_dev * dev)572 static int mt7927_wfsys_reset(struct mt792x_dev *dev)
573 {
574 struct mt76_dev *mdev = &dev->mt76;
575 u32 val;
576
577 mt7927_sema_status_read(dev, MT7927_SEMA_OWN_STA);
578
579 mt76_set(dev, MT7927_CBINFRA_RGU_WF_RST,
580 MT7927_CBINFRA_RGU_WF_RST_WF_SUBSYS);
581 usleep_range(1000, 2000);
582
583 mt76_clear(dev, MT7927_CBINFRA_RGU_WF_RST,
584 MT7927_CBINFRA_RGU_WF_RST_WF_SUBSYS);
585 usleep_range(5000, 10000);
586
587 mt76_wr(dev, MT7927_CBINFRA_MCU_OWN_SET, BIT(0));
588
589 if (!__mt76_poll_msec(mdev, MT7927_ROMCODE_INDEX, 0xffff,
590 MT7927_MCU_IDLE_VALUE, 200)) {
591 val = mt76_rr(dev, MT7927_ROMCODE_INDEX);
592 dev_err(mdev->dev,
593 "MT7927 WFSYS reset timeout (ROMCODE_INDEX=0x%04x)\n",
594 val & 0xffff);
595 return -ETIMEDOUT;
596 }
597
598 mt7927_sema_status_read(dev, MT7927_SEMA_OWN_STA_REP);
599
600 return 0;
601 }
602
mt792x_wfsys_reset_default(struct mt792x_dev * dev)603 static int mt792x_wfsys_reset_default(struct mt792x_dev *dev)
604 {
605 u32 addr = is_connac2(&dev->mt76) ? 0x18000140 : 0x7c000140;
606
607 mt76_clear(dev, addr, WFSYS_SW_RST_B);
608 msleep(50);
609 mt76_set(dev, addr, WFSYS_SW_RST_B);
610
611 if (!__mt76_poll_msec(&dev->mt76, addr, WFSYS_SW_INIT_DONE,
612 WFSYS_SW_INIT_DONE, 500))
613 return -ETIMEDOUT;
614
615 return 0;
616 }
617
mt792x_wfsys_reset(struct mt792x_dev * dev)618 int mt792x_wfsys_reset(struct mt792x_dev *dev)
619 {
620 if (is_mt7927(&dev->mt76))
621 return mt7927_wfsys_reset(dev);
622
623 return mt792x_wfsys_reset_default(dev);
624 }
625 EXPORT_SYMBOL_GPL(mt792x_wfsys_reset);
626
627