1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2018-2019 Realtek Corporation
3 */
4
5 #include <linux/dmi.h>
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include "main.h"
9 #include "pci.h"
10 #include "reg.h"
11 #include "tx.h"
12 #include "rx.h"
13 #include "fw.h"
14 #include "ps.h"
15 #include "debug.h"
16 #include "mac.h"
17
18 static bool rtw_disable_msi;
19 static bool rtw_pci_disable_aspm;
20 module_param_named(disable_msi, rtw_disable_msi, bool, 0644);
21 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644);
22 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support");
23 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support");
24
25 static const u32 rtw_pci_tx_queue_idx_addr[] = {
26 [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ,
27 [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ,
28 [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ,
29 [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ,
30 [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ,
31 [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q,
32 [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ,
33 };
34
rtw_pci_get_tx_qsel(struct sk_buff * skb,enum rtw_tx_queue_type queue)35 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb,
36 enum rtw_tx_queue_type queue)
37 {
38 switch (queue) {
39 case RTW_TX_QUEUE_BCN:
40 return TX_DESC_QSEL_BEACON;
41 case RTW_TX_QUEUE_H2C:
42 return TX_DESC_QSEL_H2C;
43 case RTW_TX_QUEUE_MGMT:
44 return TX_DESC_QSEL_MGMT;
45 case RTW_TX_QUEUE_HI0:
46 return TX_DESC_QSEL_HIGH;
47 default:
48 return skb->priority;
49 }
50 };
51
rtw_pci_read8(struct rtw_dev * rtwdev,u32 addr)52 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr)
53 {
54 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
55
56 return readb(rtwpci->mmap + addr);
57 }
58
rtw_pci_read16(struct rtw_dev * rtwdev,u32 addr)59 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr)
60 {
61 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
62
63 return readw(rtwpci->mmap + addr);
64 }
65
rtw_pci_read32(struct rtw_dev * rtwdev,u32 addr)66 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr)
67 {
68 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
69
70 return readl(rtwpci->mmap + addr);
71 }
72
rtw_pci_write8(struct rtw_dev * rtwdev,u32 addr,u8 val)73 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val)
74 {
75 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
76
77 writeb(val, rtwpci->mmap + addr);
78 }
79
rtw_pci_write16(struct rtw_dev * rtwdev,u32 addr,u16 val)80 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val)
81 {
82 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
83
84 writew(val, rtwpci->mmap + addr);
85 }
86
rtw_pci_write32(struct rtw_dev * rtwdev,u32 addr,u32 val)87 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
88 {
89 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
90
91 writel(val, rtwpci->mmap + addr);
92 }
93
rtw_pci_free_tx_ring_skbs(struct rtw_dev * rtwdev,struct rtw_pci_tx_ring * tx_ring)94 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev,
95 struct rtw_pci_tx_ring *tx_ring)
96 {
97 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
98 struct rtw_pci_tx_data *tx_data;
99 struct sk_buff *skb, *tmp;
100 dma_addr_t dma;
101
102 /* free every skb remained in tx list */
103 skb_queue_walk_safe(&tx_ring->queue, skb, tmp) {
104 __skb_unlink(skb, &tx_ring->queue);
105 tx_data = rtw_pci_get_tx_data(skb);
106 dma = tx_data->dma;
107
108 dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE);
109 dev_kfree_skb_any(skb);
110 }
111 }
112
rtw_pci_free_tx_ring(struct rtw_dev * rtwdev,struct rtw_pci_tx_ring * tx_ring)113 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev,
114 struct rtw_pci_tx_ring *tx_ring)
115 {
116 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
117 u8 *head = tx_ring->r.head;
118 u32 len = tx_ring->r.len;
119 int ring_sz = len * tx_ring->r.desc_size;
120
121 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
122
123 /* free the ring itself */
124 dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma);
125 tx_ring->r.head = NULL;
126 }
127
rtw_pci_free_rx_ring_skbs(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring)128 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev,
129 struct rtw_pci_rx_ring *rx_ring)
130 {
131 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
132 struct sk_buff *skb;
133 int buf_sz = RTK_PCI_RX_BUF_SIZE;
134 dma_addr_t dma;
135 int i;
136
137 for (i = 0; i < rx_ring->r.len; i++) {
138 skb = rx_ring->buf[i];
139 if (!skb)
140 continue;
141
142 dma = *((dma_addr_t *)skb->cb);
143 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
144 dev_kfree_skb(skb);
145 rx_ring->buf[i] = NULL;
146 }
147 }
148
rtw_pci_free_rx_ring(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring)149 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev,
150 struct rtw_pci_rx_ring *rx_ring)
151 {
152 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
153 u8 *head = rx_ring->r.head;
154 int ring_sz = rx_ring->r.desc_size * rx_ring->r.len;
155
156 rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring);
157
158 dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma);
159 }
160
rtw_pci_free_trx_ring(struct rtw_dev * rtwdev)161 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev)
162 {
163 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
164 struct rtw_pci_tx_ring *tx_ring;
165 struct rtw_pci_rx_ring *rx_ring;
166 int i;
167
168 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
169 tx_ring = &rtwpci->tx_rings[i];
170 rtw_pci_free_tx_ring(rtwdev, tx_ring);
171 }
172
173 for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) {
174 rx_ring = &rtwpci->rx_rings[i];
175 rtw_pci_free_rx_ring(rtwdev, rx_ring);
176 }
177 }
178
rtw_pci_init_tx_ring(struct rtw_dev * rtwdev,struct rtw_pci_tx_ring * tx_ring,u8 desc_size,u32 len)179 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev,
180 struct rtw_pci_tx_ring *tx_ring,
181 u8 desc_size, u32 len)
182 {
183 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
184 int ring_sz = desc_size * len;
185 dma_addr_t dma;
186 u8 *head;
187
188 if (len > TRX_BD_IDX_MASK) {
189 rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len);
190 return -EINVAL;
191 }
192
193 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
194 if (!head) {
195 rtw_err(rtwdev, "failed to allocate tx ring\n");
196 return -ENOMEM;
197 }
198
199 skb_queue_head_init(&tx_ring->queue);
200 tx_ring->r.head = head;
201 tx_ring->r.dma = dma;
202 tx_ring->r.len = len;
203 tx_ring->r.desc_size = desc_size;
204 tx_ring->r.wp = 0;
205 tx_ring->r.rp = 0;
206
207 return 0;
208 }
209
rtw_pci_reset_rx_desc(struct rtw_dev * rtwdev,struct sk_buff * skb,struct rtw_pci_rx_ring * rx_ring,u32 idx,u32 desc_sz)210 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb,
211 struct rtw_pci_rx_ring *rx_ring,
212 u32 idx, u32 desc_sz)
213 {
214 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
215 struct rtw_pci_rx_buffer_desc *buf_desc;
216 int buf_sz = RTK_PCI_RX_BUF_SIZE;
217 dma_addr_t dma;
218
219 if (!skb)
220 return -EINVAL;
221
222 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
223 if (dma_mapping_error(&pdev->dev, dma))
224 return -EBUSY;
225
226 *((dma_addr_t *)skb->cb) = dma;
227 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
228 idx * desc_sz);
229 memset(buf_desc, 0, sizeof(*buf_desc));
230 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
231 buf_desc->dma = cpu_to_le32(dma);
232
233 return 0;
234 }
235
rtw_pci_sync_rx_desc_device(struct rtw_dev * rtwdev,dma_addr_t dma,struct rtw_pci_rx_ring * rx_ring,u32 idx,u32 desc_sz)236 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma,
237 struct rtw_pci_rx_ring *rx_ring,
238 u32 idx, u32 desc_sz)
239 {
240 struct device *dev = rtwdev->dev;
241 struct rtw_pci_rx_buffer_desc *buf_desc;
242 int buf_sz = RTK_PCI_RX_BUF_SIZE;
243
244 dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE);
245
246 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
247 idx * desc_sz);
248 memset(buf_desc, 0, sizeof(*buf_desc));
249 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
250 buf_desc->dma = cpu_to_le32(dma);
251 }
252
rtw_pci_init_rx_ring(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring,u8 desc_size,u32 len)253 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev,
254 struct rtw_pci_rx_ring *rx_ring,
255 u8 desc_size, u32 len)
256 {
257 struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
258 struct sk_buff *skb = NULL;
259 dma_addr_t dma;
260 u8 *head;
261 int ring_sz = desc_size * len;
262 int buf_sz = RTK_PCI_RX_BUF_SIZE;
263 int i, allocated;
264 int ret = 0;
265
266 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
267 if (!head) {
268 rtw_err(rtwdev, "failed to allocate rx ring\n");
269 return -ENOMEM;
270 }
271 rx_ring->r.head = head;
272
273 for (i = 0; i < len; i++) {
274 skb = dev_alloc_skb(buf_sz);
275 if (!skb) {
276 allocated = i;
277 ret = -ENOMEM;
278 goto err_out;
279 }
280
281 memset(skb->data, 0, buf_sz);
282 rx_ring->buf[i] = skb;
283 ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size);
284 if (ret) {
285 allocated = i;
286 dev_kfree_skb_any(skb);
287 goto err_out;
288 }
289 }
290
291 rx_ring->r.dma = dma;
292 rx_ring->r.len = len;
293 rx_ring->r.desc_size = desc_size;
294 rx_ring->r.wp = 0;
295 rx_ring->r.rp = 0;
296
297 return 0;
298
299 err_out:
300 for (i = 0; i < allocated; i++) {
301 skb = rx_ring->buf[i];
302 if (!skb)
303 continue;
304 dma = *((dma_addr_t *)skb->cb);
305 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
306 dev_kfree_skb_any(skb);
307 rx_ring->buf[i] = NULL;
308 }
309 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
310
311 rtw_err(rtwdev, "failed to init rx buffer\n");
312
313 return ret;
314 }
315
rtw_pci_init_trx_ring(struct rtw_dev * rtwdev)316 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev)
317 {
318 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
319 struct rtw_pci_tx_ring *tx_ring;
320 struct rtw_pci_rx_ring *rx_ring;
321 const struct rtw_chip_info *chip = rtwdev->chip;
322 int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0;
323 int tx_desc_size, rx_desc_size;
324 u32 len;
325 int ret;
326
327 tx_desc_size = chip->tx_buf_desc_sz;
328
329 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
330 tx_ring = &rtwpci->tx_rings[i];
331 len = max_num_of_tx_queue(i);
332 ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len);
333 if (ret)
334 goto out;
335 }
336
337 rx_desc_size = chip->rx_buf_desc_sz;
338
339 for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) {
340 rx_ring = &rtwpci->rx_rings[j];
341 ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size,
342 RTK_MAX_RX_DESC_NUM);
343 if (ret)
344 goto out;
345 }
346
347 return 0;
348
349 out:
350 tx_alloced = i;
351 for (i = 0; i < tx_alloced; i++) {
352 tx_ring = &rtwpci->tx_rings[i];
353 rtw_pci_free_tx_ring(rtwdev, tx_ring);
354 }
355
356 rx_alloced = j;
357 for (j = 0; j < rx_alloced; j++) {
358 rx_ring = &rtwpci->rx_rings[j];
359 rtw_pci_free_rx_ring(rtwdev, rx_ring);
360 }
361
362 return ret;
363 }
364
rtw_pci_deinit(struct rtw_dev * rtwdev)365 static void rtw_pci_deinit(struct rtw_dev *rtwdev)
366 {
367 rtw_pci_free_trx_ring(rtwdev);
368 }
369
rtw_pci_init(struct rtw_dev * rtwdev)370 static int rtw_pci_init(struct rtw_dev *rtwdev)
371 {
372 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
373 int ret = 0;
374
375 rtwpci->irq_mask[0] = IMR_HIGHDOK |
376 IMR_MGNTDOK |
377 IMR_BKDOK |
378 IMR_BEDOK |
379 IMR_VIDOK |
380 IMR_VODOK |
381 IMR_ROK |
382 IMR_BCNDMAINT_E |
383 IMR_C2HCMD |
384 0;
385 rtwpci->irq_mask[1] = IMR_TXFOVW |
386 0;
387 rtwpci->irq_mask[3] = IMR_H2CDOK |
388 0;
389 spin_lock_init(&rtwpci->irq_lock);
390 spin_lock_init(&rtwpci->hwirq_lock);
391 ret = rtw_pci_init_trx_ring(rtwdev);
392
393 return ret;
394 }
395
rtw_pci_reset_buf_desc(struct rtw_dev * rtwdev)396 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)
397 {
398 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
399 u32 len;
400 u8 tmp;
401 dma_addr_t dma;
402
403 tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3);
404 rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7);
405
406 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;
407 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma);
408
409 if (!rtw_chip_wcpu_8051(rtwdev)) {
410 len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;
411 dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;
412 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;
413 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;
414 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK);
415 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma);
416 }
417
418 len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;
419 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;
420 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;
421 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;
422 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK);
423 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma);
424
425 len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;
426 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;
427 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;
428 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;
429 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK);
430 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma);
431
432 len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;
433 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;
434 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;
435 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;
436 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK);
437 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma);
438
439 len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;
440 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;
441 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;
442 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;
443 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK);
444 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma);
445
446 len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;
447 dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;
448 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;
449 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;
450 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK);
451 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma);
452
453 len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;
454 dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;
455 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;
456 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;
457 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK);
458 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma);
459
460 len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;
461 dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;
462 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;
463 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;
464 rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK);
465 rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma);
466
467 /* reset read/write point */
468 rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff);
469
470 /* reset H2C Queue index in a single write */
471 if (rtw_chip_wcpu_3081(rtwdev))
472 rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR,
473 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX);
474 }
475
rtw_pci_reset_trx_ring(struct rtw_dev * rtwdev)476 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev)
477 {
478 rtw_pci_reset_buf_desc(rtwdev);
479 }
480
rtw_pci_enable_interrupt(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,bool exclude_rx)481 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev,
482 struct rtw_pci *rtwpci, bool exclude_rx)
483 {
484 unsigned long flags;
485 u32 imr0_unmask = exclude_rx ? IMR_ROK : 0;
486
487 spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
488
489 rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);
490 rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);
491 if (rtw_chip_wcpu_3081(rtwdev))
492 rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);
493
494 rtwpci->irq_enabled = true;
495
496 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
497 }
498
rtw_pci_disable_interrupt(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)499 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev,
500 struct rtw_pci *rtwpci)
501 {
502 unsigned long flags;
503
504 spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
505
506 if (!rtwpci->irq_enabled)
507 goto out;
508
509 rtw_write32(rtwdev, RTK_PCI_HIMR0, 0);
510 rtw_write32(rtwdev, RTK_PCI_HIMR1, 0);
511 if (rtw_chip_wcpu_3081(rtwdev))
512 rtw_write32(rtwdev, RTK_PCI_HIMR3, 0);
513
514 rtwpci->irq_enabled = false;
515
516 out:
517 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
518 }
519
rtw_pci_dma_reset(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)520 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
521 {
522 /* reset dma and rx tag */
523 rtw_write32_set(rtwdev, RTK_PCI_CTRL,
524 BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN);
525 rtwpci->rx_tag = 0;
526 }
527
rtw_pci_setup(struct rtw_dev * rtwdev)528 static int rtw_pci_setup(struct rtw_dev *rtwdev)
529 {
530 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
531
532 rtw_pci_reset_trx_ring(rtwdev);
533 rtw_pci_dma_reset(rtwdev, rtwpci);
534
535 return 0;
536 }
537
rtw_pci_dma_release(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)538 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
539 {
540 struct rtw_pci_tx_ring *tx_ring;
541 enum rtw_tx_queue_type queue;
542
543 rtw_pci_reset_trx_ring(rtwdev);
544 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
545 tx_ring = &rtwpci->tx_rings[queue];
546 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
547 }
548 }
549
rtw_pci_napi_start(struct rtw_dev * rtwdev)550 static void rtw_pci_napi_start(struct rtw_dev *rtwdev)
551 {
552 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
553
554 if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
555 return;
556
557 napi_enable(&rtwpci->napi);
558 }
559
rtw_pci_napi_stop(struct rtw_dev * rtwdev)560 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev)
561 {
562 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
563
564 if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
565 return;
566
567 napi_synchronize(&rtwpci->napi);
568 napi_disable(&rtwpci->napi);
569 }
570
rtw_pci_start(struct rtw_dev * rtwdev)571 static int rtw_pci_start(struct rtw_dev *rtwdev)
572 {
573 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
574
575 rtw_pci_napi_start(rtwdev);
576
577 spin_lock_bh(&rtwpci->irq_lock);
578 rtwpci->running = true;
579 rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
580 spin_unlock_bh(&rtwpci->irq_lock);
581
582 return 0;
583 }
584
rtw_pci_stop(struct rtw_dev * rtwdev)585 static void rtw_pci_stop(struct rtw_dev *rtwdev)
586 {
587 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
588 struct pci_dev *pdev = rtwpci->pdev;
589
590 spin_lock_bh(&rtwpci->irq_lock);
591 rtwpci->running = false;
592 rtw_pci_disable_interrupt(rtwdev, rtwpci);
593 spin_unlock_bh(&rtwpci->irq_lock);
594
595 synchronize_irq(pdev->irq);
596 rtw_pci_napi_stop(rtwdev);
597
598 spin_lock_bh(&rtwpci->irq_lock);
599 rtw_pci_dma_release(rtwdev, rtwpci);
600 spin_unlock_bh(&rtwpci->irq_lock);
601 }
602
rtw_pci_deep_ps_enter(struct rtw_dev * rtwdev)603 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev)
604 {
605 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
606 struct rtw_pci_tx_ring *tx_ring;
607 enum rtw_tx_queue_type queue;
608 bool tx_empty = true;
609
610 if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
611 goto enter_deep_ps;
612
613 lockdep_assert_held(&rtwpci->irq_lock);
614
615 /* Deep PS state is not allowed to TX-DMA */
616 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
617 /* BCN queue is rsvd page, does not have DMA interrupt
618 * H2C queue is managed by firmware
619 */
620 if (queue == RTW_TX_QUEUE_BCN ||
621 queue == RTW_TX_QUEUE_H2C)
622 continue;
623
624 tx_ring = &rtwpci->tx_rings[queue];
625
626 /* check if there is any skb DMAing */
627 if (skb_queue_len(&tx_ring->queue)) {
628 tx_empty = false;
629 break;
630 }
631 }
632
633 if (!tx_empty) {
634 rtw_dbg(rtwdev, RTW_DBG_PS,
635 "TX path not empty, cannot enter deep power save state\n");
636 return;
637 }
638 enter_deep_ps:
639 set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags);
640 rtw_power_mode_change(rtwdev, true);
641 }
642
rtw_pci_deep_ps_leave(struct rtw_dev * rtwdev)643 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev)
644 {
645 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
646
647 lockdep_assert_held(&rtwpci->irq_lock);
648
649 if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
650 rtw_power_mode_change(rtwdev, false);
651 }
652
rtw_pci_deep_ps(struct rtw_dev * rtwdev,bool enter)653 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter)
654 {
655 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
656
657 spin_lock_bh(&rtwpci->irq_lock);
658
659 if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
660 rtw_pci_deep_ps_enter(rtwdev);
661
662 if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
663 rtw_pci_deep_ps_leave(rtwdev);
664
665 spin_unlock_bh(&rtwpci->irq_lock);
666 }
667
rtw_pci_release_rsvd_page(struct rtw_pci * rtwpci,struct rtw_pci_tx_ring * ring)668 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,
669 struct rtw_pci_tx_ring *ring)
670 {
671 struct sk_buff *prev = skb_dequeue(&ring->queue);
672 struct rtw_pci_tx_data *tx_data;
673 dma_addr_t dma;
674
675 if (!prev)
676 return;
677
678 tx_data = rtw_pci_get_tx_data(prev);
679 dma = tx_data->dma;
680 dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);
681 dev_kfree_skb_any(prev);
682 }
683
rtw_pci_dma_check(struct rtw_dev * rtwdev,struct rtw_pci_rx_ring * rx_ring,u32 idx)684 static void rtw_pci_dma_check(struct rtw_dev *rtwdev,
685 struct rtw_pci_rx_ring *rx_ring,
686 u32 idx)
687 {
688 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
689 const struct rtw_chip_info *chip = rtwdev->chip;
690 struct rtw_pci_rx_buffer_desc *buf_desc;
691 u32 desc_sz = chip->rx_buf_desc_sz;
692 u16 total_pkt_size;
693
694 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
695 idx * desc_sz);
696 total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size);
697
698 /* rx tag mismatch, throw a warning */
699 if (total_pkt_size != rtwpci->rx_tag)
700 rtw_warn(rtwdev, "pci bus timeout, check dma status\n");
701
702 rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;
703 }
704
__pci_get_hw_tx_ring_rp(struct rtw_dev * rtwdev,u8 pci_q)705 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q)
706 {
707 u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q];
708 u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2);
709
710 return FIELD_GET(TRX_BD_IDX_MASK, bd_idx);
711 }
712
__pci_flush_queue(struct rtw_dev * rtwdev,u8 pci_q,bool drop)713 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop)
714 {
715 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
716 struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];
717 u32 cur_rp;
718 u8 i;
719
720 /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a
721 * bit dynamic, it's hard to define a reasonable fixed total timeout to
722 * use read_poll_timeout* helper. Instead, we can ensure a reasonable
723 * polling times, so we just use for loop with udelay here.
724 */
725 for (i = 0; i < 30; i++) {
726 cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q);
727 if (cur_rp == ring->r.wp)
728 return;
729
730 udelay(1);
731 }
732
733 if (!drop)
734 rtw_dbg(rtwdev, RTW_DBG_UNEXP,
735 "timed out to flush pci tx ring[%d]\n", pci_q);
736 }
737
__rtw_pci_flush_queues(struct rtw_dev * rtwdev,u32 pci_queues,bool drop)738 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues,
739 bool drop)
740 {
741 u8 q;
742
743 for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) {
744 /* Unnecessary to flush BCN, H2C and HI tx queues. */
745 if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C ||
746 q == RTW_TX_QUEUE_HI0)
747 continue;
748
749 if (pci_queues & BIT(q))
750 __pci_flush_queue(rtwdev, q, drop);
751 }
752 }
753
rtw_pci_flush_queues(struct rtw_dev * rtwdev,u32 queues,bool drop)754 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop)
755 {
756 u32 pci_queues = 0;
757 u8 i;
758
759 /* If all of the hardware queues are requested to flush,
760 * flush all of the pci queues.
761 */
762 if (queues == BIT(rtwdev->hw->queues) - 1) {
763 pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1;
764 } else {
765 for (i = 0; i < rtwdev->hw->queues; i++)
766 if (queues & BIT(i))
767 pci_queues |= BIT(rtw_tx_ac_to_hwq(i));
768 }
769
770 __rtw_pci_flush_queues(rtwdev, pci_queues, drop);
771 }
772
rtw_pci_tx_kick_off_queue(struct rtw_dev * rtwdev,enum rtw_tx_queue_type queue)773 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev,
774 enum rtw_tx_queue_type queue)
775 {
776 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
777 struct rtw_pci_tx_ring *ring;
778 u32 bd_idx;
779
780 ring = &rtwpci->tx_rings[queue];
781 bd_idx = rtw_pci_tx_queue_idx_addr[queue];
782
783 spin_lock_bh(&rtwpci->irq_lock);
784 if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
785 rtw_pci_deep_ps_leave(rtwdev);
786 rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK);
787 spin_unlock_bh(&rtwpci->irq_lock);
788 }
789
rtw_pci_tx_kick_off(struct rtw_dev * rtwdev)790 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev)
791 {
792 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
793 enum rtw_tx_queue_type queue;
794
795 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++)
796 if (test_and_clear_bit(queue, rtwpci->tx_queued))
797 rtw_pci_tx_kick_off_queue(rtwdev, queue);
798 }
799
rtw_pci_tx_write_data(struct rtw_dev * rtwdev,struct rtw_tx_pkt_info * pkt_info,struct sk_buff * skb,enum rtw_tx_queue_type queue)800 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev,
801 struct rtw_tx_pkt_info *pkt_info,
802 struct sk_buff *skb,
803 enum rtw_tx_queue_type queue)
804 {
805 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
806 const struct rtw_chip_info *chip = rtwdev->chip;
807 struct rtw_pci_tx_ring *ring;
808 struct rtw_pci_tx_data *tx_data;
809 dma_addr_t dma;
810 u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz;
811 u32 tx_buf_desc_sz = chip->tx_buf_desc_sz;
812 u32 size;
813 u32 psb_len;
814 u8 *pkt_desc;
815 struct rtw_pci_tx_buffer_desc *buf_desc;
816
817 ring = &rtwpci->tx_rings[queue];
818
819 size = skb->len;
820
821 if (queue == RTW_TX_QUEUE_BCN)
822 rtw_pci_release_rsvd_page(rtwpci, ring);
823 else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len))
824 return -ENOSPC;
825
826 pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz);
827 memset(pkt_desc, 0, tx_pkt_desc_sz);
828 pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue);
829 rtw_tx_fill_tx_desc(rtwdev, pkt_info, skb);
830 dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,
831 DMA_TO_DEVICE);
832 if (dma_mapping_error(&rtwpci->pdev->dev, dma))
833 return -EBUSY;
834
835 /* after this we got dma mapped, there is no way back */
836 buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz);
837 memset(buf_desc, 0, tx_buf_desc_sz);
838 psb_len = (skb->len - 1) / 128 + 1;
839 if (queue == RTW_TX_QUEUE_BCN)
840 psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET;
841
842 buf_desc[0].psb_len = cpu_to_le16(psb_len);
843 buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz);
844 buf_desc[0].dma = cpu_to_le32(dma);
845 buf_desc[1].buf_size = cpu_to_le16(size);
846 buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz);
847
848 tx_data = rtw_pci_get_tx_data(skb);
849 tx_data->dma = dma;
850 tx_data->sn = pkt_info->sn;
851
852 spin_lock_bh(&rtwpci->irq_lock);
853
854 skb_queue_tail(&ring->queue, skb);
855
856 if (queue == RTW_TX_QUEUE_BCN)
857 goto out_unlock;
858
859 /* update write-index, and kick it off later */
860 set_bit(queue, rtwpci->tx_queued);
861 if (++ring->r.wp >= ring->r.len)
862 ring->r.wp = 0;
863
864 out_unlock:
865 spin_unlock_bh(&rtwpci->irq_lock);
866
867 return 0;
868 }
869
rtw_pci_write_data_rsvd_page(struct rtw_dev * rtwdev,u8 * buf,u32 size)870 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf,
871 u32 size)
872 {
873 struct sk_buff *skb;
874 struct rtw_tx_pkt_info pkt_info = {0};
875 u8 reg_bcn_work;
876 int ret;
877
878 skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size);
879 if (!skb)
880 return -ENOMEM;
881
882 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN);
883 if (ret) {
884 rtw_err(rtwdev, "failed to write rsvd page data\n");
885 return ret;
886 }
887
888 /* reserved pages go through beacon queue */
889 reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK);
890 reg_bcn_work |= BIT_PCI_BCNQ_FLAG;
891 rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work);
892
893 return 0;
894 }
895
rtw_pci_write_data_h2c(struct rtw_dev * rtwdev,u8 * buf,u32 size)896 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size)
897 {
898 struct sk_buff *skb;
899 struct rtw_tx_pkt_info pkt_info = {0};
900 int ret;
901
902 skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size);
903 if (!skb)
904 return -ENOMEM;
905
906 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C);
907 if (ret) {
908 rtw_err(rtwdev, "failed to write h2c data\n");
909 return ret;
910 }
911
912 rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C);
913
914 return 0;
915 }
916
rtw_pci_tx_write(struct rtw_dev * rtwdev,struct rtw_tx_pkt_info * pkt_info,struct sk_buff * skb)917 static int rtw_pci_tx_write(struct rtw_dev *rtwdev,
918 struct rtw_tx_pkt_info *pkt_info,
919 struct sk_buff *skb)
920 {
921 enum rtw_tx_queue_type queue = rtw_tx_queue_mapping(skb);
922 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
923 struct rtw_pci_tx_ring *ring;
924 int ret;
925
926 ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue);
927 if (ret)
928 return ret;
929
930 ring = &rtwpci->tx_rings[queue];
931 spin_lock_bh(&rtwpci->irq_lock);
932 if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) {
933 ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb));
934 ring->queue_stopped = true;
935 }
936 spin_unlock_bh(&rtwpci->irq_lock);
937
938 return 0;
939 }
940
rtw_pci_tx_isr(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,u8 hw_queue)941 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
942 u8 hw_queue)
943 {
944 struct ieee80211_hw *hw = rtwdev->hw;
945 struct ieee80211_tx_info *info;
946 struct rtw_pci_tx_ring *ring;
947 struct rtw_pci_tx_data *tx_data;
948 struct sk_buff *skb;
949 u32 count;
950 u32 bd_idx_addr;
951 u32 bd_idx, cur_rp, rp_idx;
952 u16 q_map;
953
954 ring = &rtwpci->tx_rings[hw_queue];
955
956 bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue];
957 bd_idx = rtw_read32(rtwdev, bd_idx_addr);
958 cur_rp = bd_idx >> 16;
959 cur_rp &= TRX_BD_IDX_MASK;
960 rp_idx = ring->r.rp;
961 if (cur_rp >= ring->r.rp)
962 count = cur_rp - ring->r.rp;
963 else
964 count = ring->r.len - (ring->r.rp - cur_rp);
965
966 while (count--) {
967 skb = skb_dequeue(&ring->queue);
968 if (!skb) {
969 rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n",
970 count, hw_queue, bd_idx, ring->r.rp, cur_rp);
971 break;
972 }
973 tx_data = rtw_pci_get_tx_data(skb);
974 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
975 DMA_TO_DEVICE);
976
977 /* just free command packets from host to card */
978 if (hw_queue == RTW_TX_QUEUE_H2C) {
979 dev_kfree_skb_irq(skb);
980 continue;
981 }
982
983 if (ring->queue_stopped &&
984 avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) {
985 q_map = skb_get_queue_mapping(skb);
986 ieee80211_wake_queue(hw, q_map);
987 ring->queue_stopped = false;
988 }
989
990 if (++rp_idx >= ring->r.len)
991 rp_idx = 0;
992
993 skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz);
994
995 info = IEEE80211_SKB_CB(skb);
996
997 /* enqueue to wait for tx report */
998 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
999 rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn);
1000 continue;
1001 }
1002
1003 /* always ACK for others, then they won't be marked as drop */
1004 if (info->flags & IEEE80211_TX_CTL_NO_ACK)
1005 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
1006 else
1007 info->flags |= IEEE80211_TX_STAT_ACK;
1008
1009 ieee80211_tx_info_clear_status(info);
1010 ieee80211_tx_status_irqsafe(hw, skb);
1011 }
1012
1013 ring->r.rp = cur_rp;
1014 }
1015
rtw_pci_rx_isr(struct rtw_dev * rtwdev)1016 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev)
1017 {
1018 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1019 struct napi_struct *napi = &rtwpci->napi;
1020
1021 napi_schedule(napi);
1022 }
1023
rtw_pci_get_hw_rx_ring_nr(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci)1024 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev,
1025 struct rtw_pci *rtwpci)
1026 {
1027 struct rtw_pci_rx_ring *ring;
1028 int count = 0;
1029 u32 tmp, cur_wp;
1030
1031 ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1032 tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ);
1033 cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK);
1034 if (cur_wp >= ring->r.wp)
1035 count = cur_wp - ring->r.wp;
1036 else
1037 count = ring->r.len - (ring->r.wp - cur_wp);
1038
1039 return count;
1040 }
1041
rtw_pci_rx_napi(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,u8 hw_queue,u32 limit)1042 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
1043 u8 hw_queue, u32 limit)
1044 {
1045 struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1046 const struct rtw_chip_info *chip = rtwdev->chip;
1047 struct napi_struct *napi = &rtwpci->napi;
1048 u32 pkt_desc_sz = chip->rx_pkt_desc_sz;
1049 u32 buf_desc_sz = chip->rx_buf_desc_sz;
1050 struct ieee80211_rx_status rx_status;
1051 struct rtw_rx_pkt_stat pkt_stat;
1052 struct sk_buff *skb, *new;
1053 u32 cur_rp = ring->r.rp;
1054 u32 count, rx_done = 0;
1055 u32 pkt_offset;
1056 dma_addr_t dma;
1057 u32 new_len;
1058 u8 *rx_desc;
1059 int ret;
1060
1061 count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);
1062 count = min(count, limit);
1063
1064 while (count--) {
1065 rtw_pci_dma_check(rtwdev, ring, cur_rp);
1066 skb = ring->buf[cur_rp];
1067 dma = *((dma_addr_t *)skb->cb);
1068 dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE,
1069 DMA_FROM_DEVICE);
1070 rx_desc = skb->data;
1071 ret = rtw_rx_query_rx_desc(rtwdev, rx_desc,
1072 &pkt_stat, &rx_status);
1073 if (ret)
1074 goto next_rp;
1075
1076 /* offset from rx_desc to payload */
1077 pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz +
1078 pkt_stat.shift;
1079
1080 /* allocate a new skb for this frame,
1081 * discard the frame if none available
1082 */
1083 new_len = pkt_stat.pkt_len + pkt_offset;
1084 if (unlikely(new_len > RTK_PCI_RX_BUF_SIZE)) {
1085 rtw_dbg(rtwdev, RTW_DBG_RX,
1086 "oversized RX packet: %u\n", new_len);
1087 goto next_rp;
1088 }
1089 new = dev_alloc_skb(new_len);
1090 if (WARN_ONCE(!new, "rx routine starvation\n"))
1091 goto next_rp;
1092
1093 /* put the DMA data including rx_desc from phy to new skb */
1094 skb_put_data(new, skb->data, new_len);
1095
1096 if (pkt_stat.is_c2h) {
1097 rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new);
1098 } else {
1099 /* remove rx_desc */
1100 skb_pull(new, pkt_offset);
1101
1102 rtw_update_rx_freq_for_invalid(rtwdev, new, &rx_status, &pkt_stat);
1103 rtw_rx_stats(rtwdev, pkt_stat.vif, new);
1104 memcpy(new->cb, &rx_status, sizeof(rx_status));
1105 ieee80211_rx_napi(rtwdev->hw, NULL, new, napi);
1106 rx_done++;
1107 }
1108
1109 next_rp:
1110 /* new skb delivered to mac80211, re-enable original skb DMA */
1111 rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp,
1112 buf_desc_sz);
1113
1114 /* host read next element in ring */
1115 if (++cur_rp >= ring->r.len)
1116 cur_rp = 0;
1117 }
1118
1119 ring->r.rp = cur_rp;
1120 /* 'rp', the last position we have read, is seen as previous posistion
1121 * of 'wp' that is used to calculate 'count' next time.
1122 */
1123 ring->r.wp = cur_rp;
1124 rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp);
1125
1126 return rx_done;
1127 }
1128
rtw_pci_irq_recognized(struct rtw_dev * rtwdev,struct rtw_pci * rtwpci,u32 * irq_status)1129 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,
1130 struct rtw_pci *rtwpci, u32 *irq_status)
1131 {
1132 unsigned long flags;
1133
1134 spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
1135
1136 irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0);
1137 irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1);
1138 if (rtw_chip_wcpu_3081(rtwdev))
1139 irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3);
1140 else
1141 irq_status[3] = 0;
1142 irq_status[0] &= rtwpci->irq_mask[0];
1143 irq_status[1] &= rtwpci->irq_mask[1];
1144 irq_status[3] &= rtwpci->irq_mask[3];
1145 rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]);
1146 rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]);
1147 if (rtw_chip_wcpu_3081(rtwdev))
1148 rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]);
1149
1150 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
1151 }
1152
rtw_pci_interrupt_handler(int irq,void * dev)1153 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev)
1154 {
1155 struct rtw_dev *rtwdev = dev;
1156 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1157
1158 /* disable RTW PCI interrupt to avoid more interrupts before the end of
1159 * thread function
1160 *
1161 * disable HIMR here to also avoid new HISR flag being raised before
1162 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs
1163 * are cleared, the edge-triggered interrupt will not be generated when
1164 * a new HISR flag is set.
1165 */
1166 rtw_pci_disable_interrupt(rtwdev, rtwpci);
1167
1168 return IRQ_WAKE_THREAD;
1169 }
1170
rtw_pci_interrupt_threadfn(int irq,void * dev)1171 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev)
1172 {
1173 struct rtw_dev *rtwdev = dev;
1174 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1175 u32 irq_status[4];
1176 bool rx = false;
1177
1178 spin_lock_bh(&rtwpci->irq_lock);
1179 rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);
1180
1181 if (irq_status[0] & IMR_MGNTDOK)
1182 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);
1183 if (irq_status[0] & IMR_HIGHDOK)
1184 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);
1185 if (irq_status[0] & IMR_BEDOK)
1186 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);
1187 if (irq_status[0] & IMR_BKDOK)
1188 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);
1189 if (irq_status[0] & IMR_VODOK)
1190 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);
1191 if (irq_status[0] & IMR_VIDOK)
1192 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);
1193 if (irq_status[3] & IMR_H2CDOK)
1194 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);
1195 if (irq_status[0] & IMR_ROK) {
1196 rtw_pci_rx_isr(rtwdev);
1197 rx = true;
1198 }
1199 if (unlikely(irq_status[0] & IMR_C2HCMD))
1200 rtw_fw_c2h_cmd_isr(rtwdev);
1201
1202 /* all of the jobs for this interrupt have been done */
1203 if (rtwpci->running)
1204 rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);
1205 spin_unlock_bh(&rtwpci->irq_lock);
1206
1207 return IRQ_HANDLED;
1208 }
1209
rtw_pci_io_mapping(struct rtw_dev * rtwdev,struct pci_dev * pdev)1210 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev,
1211 struct pci_dev *pdev)
1212 {
1213 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1214 unsigned long len;
1215 u8 bar_id = 2;
1216 int ret;
1217
1218 ret = pci_request_regions(pdev, KBUILD_MODNAME);
1219 if (ret) {
1220 rtw_err(rtwdev, "failed to request pci regions\n");
1221 return ret;
1222 }
1223
1224 len = pci_resource_len(pdev, bar_id);
1225 rtwpci->mmap = pci_iomap(pdev, bar_id, len);
1226 if (!rtwpci->mmap) {
1227 pci_release_regions(pdev);
1228 rtw_err(rtwdev, "failed to map pci memory\n");
1229 return -ENOMEM;
1230 }
1231
1232 return 0;
1233 }
1234
rtw_pci_io_unmapping(struct rtw_dev * rtwdev,struct pci_dev * pdev)1235 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev,
1236 struct pci_dev *pdev)
1237 {
1238 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1239
1240 if (rtwpci->mmap) {
1241 pci_iounmap(pdev, rtwpci->mmap);
1242 pci_release_regions(pdev);
1243 }
1244 }
1245
rtw_dbi_write8(struct rtw_dev * rtwdev,u16 addr,u8 data)1246 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data)
1247 {
1248 u16 write_addr;
1249 u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK);
1250 u8 flag;
1251 u8 cnt;
1252
1253 write_addr = addr & BITS_DBI_ADDR_MASK;
1254 write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN);
1255 rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data);
1256 rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr);
1257 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16);
1258
1259 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1260 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1261 if (flag == 0)
1262 return;
1263
1264 udelay(10);
1265 }
1266
1267 WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr);
1268 }
1269
rtw_dbi_read8(struct rtw_dev * rtwdev,u16 addr,u8 * value)1270 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value)
1271 {
1272 u16 read_addr = addr & BITS_DBI_ADDR_MASK;
1273 u8 flag;
1274 u8 cnt;
1275
1276 rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr);
1277 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16);
1278
1279 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1280 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1281 if (flag == 0) {
1282 read_addr = REG_DBI_RDATA_V1 + (addr & 3);
1283 *value = rtw_read8(rtwdev, read_addr);
1284 return 0;
1285 }
1286
1287 udelay(10);
1288 }
1289
1290 WARN(1, "failed to read DBI register, addr=0x%04x\n", addr);
1291 return -EIO;
1292 }
1293
rtw_mdio_write(struct rtw_dev * rtwdev,u8 addr,u16 data,bool g1)1294 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1)
1295 {
1296 u8 page;
1297 u8 wflag;
1298 u8 cnt;
1299
1300 rtw_write16(rtwdev, REG_MDIO_V1, data);
1301
1302 page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1;
1303 page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2;
1304 rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK);
1305 rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page);
1306 rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1);
1307
1308 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1309 wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG,
1310 BIT_MDIO_WFLAG_V1);
1311 if (wflag == 0)
1312 return;
1313
1314 udelay(10);
1315 }
1316
1317 WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr);
1318 }
1319
rtw_pci_clkreq_set(struct rtw_dev * rtwdev,bool enable)1320 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable)
1321 {
1322 u8 value;
1323 int ret;
1324
1325 if (rtw_pci_disable_aspm)
1326 return;
1327
1328 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1329 if (ret) {
1330 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1331 return;
1332 }
1333
1334 if (enable)
1335 value |= BIT_CLKREQ_SW_EN;
1336 else
1337 value &= ~BIT_CLKREQ_SW_EN;
1338
1339 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1340 }
1341
rtw_pci_clkreq_pad_low(struct rtw_dev * rtwdev,bool enable)1342 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable)
1343 {
1344 u8 value;
1345 int ret;
1346
1347 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1348 if (ret) {
1349 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1350 return;
1351 }
1352
1353 if (enable)
1354 value &= ~BIT_CLKREQ_N_PAD;
1355 else
1356 value |= BIT_CLKREQ_N_PAD;
1357
1358 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1359 }
1360
rtw_pci_aspm_set(struct rtw_dev * rtwdev,bool enable)1361 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable)
1362 {
1363 u8 value;
1364 int ret;
1365
1366 if (rtw_pci_disable_aspm)
1367 return;
1368
1369 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1370 if (ret) {
1371 rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret);
1372 return;
1373 }
1374
1375 if (enable)
1376 value |= BIT_L1_SW_EN;
1377 else
1378 value &= ~BIT_L1_SW_EN;
1379
1380 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1381 }
1382
rtw_pci_link_ps(struct rtw_dev * rtwdev,bool enter)1383 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)
1384 {
1385 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1386
1387 /* Like CLKREQ, ASPM is also implemented by two HW modules, and can
1388 * only be enabled when host supports it.
1389 *
1390 * And ASPM mechanism should be enabled when driver/firmware enters
1391 * power save mode, without having heavy traffic. Because we've
1392 * experienced some inter-operability issues that the link tends
1393 * to enter L1 state on the fly even when driver is having high
1394 * throughput. This is probably because the ASPM behavior slightly
1395 * varies from different SOC.
1396 */
1397 if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))
1398 return;
1399
1400 if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||
1401 (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))
1402 rtw_pci_aspm_set(rtwdev, enter);
1403 }
1404
rtw_pci_link_cfg(struct rtw_dev * rtwdev)1405 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)
1406 {
1407 const struct rtw_chip_info *chip = rtwdev->chip;
1408 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1409 struct pci_dev *pdev = rtwpci->pdev;
1410 u16 link_ctrl;
1411 int ret;
1412
1413 /* RTL8822CE has enabled REFCLK auto calibration, it does not need
1414 * to add clock delay to cover the REFCLK timing gap.
1415 */
1416 if (chip->id == RTW_CHIP_TYPE_8822C)
1417 rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0);
1418
1419 /* Though there is standard PCIE configuration space to set the
1420 * link control register, but by Realtek's design, driver should
1421 * check if host supports CLKREQ/ASPM to enable the HW module.
1422 *
1423 * These functions are implemented by two HW modules associated,
1424 * one is responsible to access PCIE configuration space to
1425 * follow the host settings, and another is in charge of doing
1426 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
1427 * the host does not support it, and due to some reasons or wrong
1428 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
1429 * loss if HW misbehaves on the link.
1430 *
1431 * Hence it's designed that driver should first check the PCIE
1432 * configuration space is sync'ed and enabled, then driver can turn
1433 * on the other module that is actually working on the mechanism.
1434 */
1435 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
1436 if (ret) {
1437 rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
1438 return;
1439 }
1440
1441 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
1442 rtw_pci_clkreq_set(rtwdev, true);
1443
1444 rtwpci->link_ctrl = link_ctrl;
1445 }
1446
rtw_pci_interface_cfg(struct rtw_dev * rtwdev)1447 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)
1448 {
1449 const struct rtw_chip_info *chip = rtwdev->chip;
1450
1451 switch (chip->id) {
1452 case RTW_CHIP_TYPE_8822C:
1453 if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D)
1454 rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG,
1455 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1);
1456 break;
1457 default:
1458 break;
1459 }
1460 }
1461
rtw_pci_phy_cfg(struct rtw_dev * rtwdev)1462 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev)
1463 {
1464 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1465 const struct rtw_chip_info *chip = rtwdev->chip;
1466 struct rtw_efuse *efuse = &rtwdev->efuse;
1467 struct pci_dev *pdev = rtwpci->pdev;
1468 const struct rtw_intf_phy_para *para;
1469 u16 cut;
1470 u16 value;
1471 u16 offset;
1472 int i;
1473 int ret;
1474
1475 cut = BIT(0) << rtwdev->hal.cut_version;
1476
1477 for (i = 0; i < chip->intf_table->n_gen1_para; i++) {
1478 para = &chip->intf_table->gen1_para[i];
1479 if (!(para->cut_mask & cut))
1480 continue;
1481 if (para->offset == 0xffff)
1482 break;
1483 offset = para->offset;
1484 value = para->value;
1485 if (para->ip_sel == RTW_IP_SEL_PHY)
1486 rtw_mdio_write(rtwdev, offset, value, true);
1487 else
1488 rtw_dbi_write8(rtwdev, offset, value);
1489 }
1490
1491 for (i = 0; i < chip->intf_table->n_gen2_para; i++) {
1492 para = &chip->intf_table->gen2_para[i];
1493 if (!(para->cut_mask & cut))
1494 continue;
1495 if (para->offset == 0xffff)
1496 break;
1497 offset = para->offset;
1498 value = para->value;
1499 if (para->ip_sel == RTW_IP_SEL_PHY)
1500 rtw_mdio_write(rtwdev, offset, value, false);
1501 else
1502 rtw_dbi_write8(rtwdev, offset, value);
1503 }
1504
1505 rtw_pci_link_cfg(rtwdev);
1506
1507 /* Disable 8821ce completion timeout by default */
1508 if (chip->id == RTW_CHIP_TYPE_8821C) {
1509 ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
1510 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS);
1511 if (ret)
1512 rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n",
1513 ret);
1514 }
1515
1516 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 5)
1517 rtw_write32_mask(rtwdev, REG_ANAPARSW_MAC_0, BIT_CF_L_V2, 0x1);
1518 }
1519
rtw_pci_suspend(struct device * dev)1520 static int __maybe_unused rtw_pci_suspend(struct device *dev)
1521 {
1522 struct ieee80211_hw *hw = dev_get_drvdata(dev);
1523 struct rtw_dev *rtwdev = hw->priv;
1524 const struct rtw_chip_info *chip = rtwdev->chip;
1525 struct rtw_efuse *efuse = &rtwdev->efuse;
1526
1527 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1528 rtw_pci_clkreq_pad_low(rtwdev, true);
1529 return 0;
1530 }
1531
rtw_pci_resume(struct device * dev)1532 static int __maybe_unused rtw_pci_resume(struct device *dev)
1533 {
1534 struct ieee80211_hw *hw = dev_get_drvdata(dev);
1535 struct rtw_dev *rtwdev = hw->priv;
1536 const struct rtw_chip_info *chip = rtwdev->chip;
1537 struct rtw_efuse *efuse = &rtwdev->efuse;
1538
1539 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1540 rtw_pci_clkreq_pad_low(rtwdev, false);
1541 return 0;
1542 }
1543
1544 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume);
1545 EXPORT_SYMBOL(rtw_pm_ops);
1546
rtw_pci_claim(struct rtw_dev * rtwdev,struct pci_dev * pdev)1547 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1548 {
1549 int ret;
1550
1551 ret = pci_enable_device(pdev);
1552 if (ret) {
1553 rtw_err(rtwdev, "failed to enable pci device\n");
1554 return ret;
1555 }
1556
1557 pci_set_master(pdev);
1558 pci_set_drvdata(pdev, rtwdev->hw);
1559 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
1560
1561 return 0;
1562 }
1563
rtw_pci_declaim(struct rtw_dev * rtwdev,struct pci_dev * pdev)1564 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1565 {
1566 pci_disable_device(pdev);
1567 }
1568
rtw_pci_setup_resource(struct rtw_dev * rtwdev,struct pci_dev * pdev)1569 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1570 {
1571 struct rtw_pci *rtwpci;
1572 int ret;
1573
1574 rtwpci = (struct rtw_pci *)rtwdev->priv;
1575 rtwpci->pdev = pdev;
1576
1577 /* after this driver can access to hw registers */
1578 ret = rtw_pci_io_mapping(rtwdev, pdev);
1579 if (ret) {
1580 rtw_err(rtwdev, "failed to request pci io region\n");
1581 goto err_out;
1582 }
1583
1584 ret = rtw_pci_init(rtwdev);
1585 if (ret) {
1586 rtw_err(rtwdev, "failed to allocate pci resources\n");
1587 goto err_io_unmap;
1588 }
1589
1590 return 0;
1591
1592 err_io_unmap:
1593 rtw_pci_io_unmapping(rtwdev, pdev);
1594
1595 err_out:
1596 return ret;
1597 }
1598
rtw_pci_destroy(struct rtw_dev * rtwdev,struct pci_dev * pdev)1599 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1600 {
1601 rtw_pci_deinit(rtwdev);
1602 rtw_pci_io_unmapping(rtwdev, pdev);
1603 }
1604
1605 static const struct rtw_hci_ops rtw_pci_ops = {
1606 .tx_write = rtw_pci_tx_write,
1607 .tx_kick_off = rtw_pci_tx_kick_off,
1608 .flush_queues = rtw_pci_flush_queues,
1609 .setup = rtw_pci_setup,
1610 .start = rtw_pci_start,
1611 .stop = rtw_pci_stop,
1612 .deep_ps = rtw_pci_deep_ps,
1613 .link_ps = rtw_pci_link_ps,
1614 .interface_cfg = rtw_pci_interface_cfg,
1615 .dynamic_rx_agg = NULL,
1616 .write_firmware_page = rtw_write_firmware_page,
1617
1618 .read8 = rtw_pci_read8,
1619 .read16 = rtw_pci_read16,
1620 .read32 = rtw_pci_read32,
1621 .write8 = rtw_pci_write8,
1622 .write16 = rtw_pci_write16,
1623 .write32 = rtw_pci_write32,
1624 .write_data_rsvd_page = rtw_pci_write_data_rsvd_page,
1625 .write_data_h2c = rtw_pci_write_data_h2c,
1626 };
1627
rtw_pci_request_irq(struct rtw_dev * rtwdev,struct pci_dev * pdev)1628 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1629 {
1630 unsigned int flags = PCI_IRQ_INTX;
1631 int ret;
1632
1633 if (!rtw_disable_msi)
1634 flags |= PCI_IRQ_MSI;
1635
1636 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
1637 if (ret < 0) {
1638 rtw_err(rtwdev, "failed to alloc PCI irq vectors\n");
1639 return ret;
1640 }
1641
1642 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
1643 rtw_pci_interrupt_handler,
1644 rtw_pci_interrupt_threadfn,
1645 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
1646 if (ret) {
1647 rtw_err(rtwdev, "failed to request irq %d\n", ret);
1648 pci_free_irq_vectors(pdev);
1649 }
1650
1651 return ret;
1652 }
1653
rtw_pci_free_irq(struct rtw_dev * rtwdev,struct pci_dev * pdev)1654 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1655 {
1656 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
1657 pci_free_irq_vectors(pdev);
1658 }
1659
rtw_pci_napi_poll(struct napi_struct * napi,int budget)1660 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget)
1661 {
1662 struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);
1663 struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,
1664 priv);
1665 int work_done = 0;
1666
1667 if (rtwpci->rx_no_aspm)
1668 rtw_pci_link_ps(rtwdev, false);
1669
1670 while (work_done < budget) {
1671 u32 work_done_once;
1672
1673 work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,
1674 budget - work_done);
1675 if (work_done_once == 0)
1676 break;
1677 work_done += work_done_once;
1678 }
1679 if (work_done < budget) {
1680 napi_complete_done(napi, work_done);
1681 spin_lock_bh(&rtwpci->irq_lock);
1682 if (rtwpci->running)
1683 rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
1684 spin_unlock_bh(&rtwpci->irq_lock);
1685 /* When ISR happens during polling and before napi_complete
1686 * while no further data is received. Data on the dma_ring will
1687 * not be processed immediately. Check whether dma ring is
1688 * empty and perform napi_schedule accordingly.
1689 */
1690 if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))
1691 napi_schedule(napi);
1692 }
1693 if (rtwpci->rx_no_aspm)
1694 rtw_pci_link_ps(rtwdev, true);
1695
1696 return work_done;
1697 }
1698
rtw_pci_napi_init(struct rtw_dev * rtwdev)1699 static int rtw_pci_napi_init(struct rtw_dev *rtwdev)
1700 {
1701 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1702
1703 rtwpci->netdev = alloc_netdev_dummy(0);
1704 if (!rtwpci->netdev)
1705 return -ENOMEM;
1706
1707 netif_napi_add(rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);
1708 return 0;
1709 }
1710
rtw_pci_napi_deinit(struct rtw_dev * rtwdev)1711 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
1712 {
1713 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1714
1715 rtw_pci_napi_stop(rtwdev);
1716 netif_napi_del(&rtwpci->napi);
1717 free_netdev(rtwpci->netdev);
1718 }
1719
rtw_pci_io_err_detected(struct pci_dev * pdev,pci_channel_state_t state)1720 static pci_ers_result_t rtw_pci_io_err_detected(struct pci_dev *pdev,
1721 pci_channel_state_t state)
1722 {
1723 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1724
1725 ieee80211_stop_queues(hw);
1726
1727 return PCI_ERS_RESULT_NEED_RESET;
1728 }
1729
rtw_pci_io_slot_reset(struct pci_dev * pdev)1730 static pci_ers_result_t rtw_pci_io_slot_reset(struct pci_dev *pdev)
1731 {
1732 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1733 struct rtw_dev *rtwdev = hw->priv;
1734
1735 rtw_fw_recovery(rtwdev);
1736
1737 return PCI_ERS_RESULT_RECOVERED;
1738 }
1739
rtw_pci_io_resume(struct pci_dev * pdev)1740 static void rtw_pci_io_resume(struct pci_dev *pdev)
1741 {
1742 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1743
1744 /* ack any pending wake events, disable PME */
1745 pci_enable_wake(pdev, PCI_D0, 0);
1746
1747 ieee80211_wake_queues(hw);
1748 }
1749
1750 const struct pci_error_handlers rtw_pci_err_handler = {
1751 .error_detected = rtw_pci_io_err_detected,
1752 .slot_reset = rtw_pci_io_slot_reset,
1753 .resume = rtw_pci_io_resume,
1754 };
1755 EXPORT_SYMBOL(rtw_pci_err_handler);
1756
rtw_pci_disable_caps(const struct dmi_system_id * dmi)1757 static int rtw_pci_disable_caps(const struct dmi_system_id *dmi)
1758 {
1759 uintptr_t dis_caps = (uintptr_t)dmi->driver_data;
1760
1761 if (dis_caps & BIT(QUIRK_DIS_CAP_PCI_ASPM))
1762 rtw_pci_disable_aspm = true;
1763
1764 if (dis_caps & BIT(QUIRK_DIS_CAP_LPS_DEEP))
1765 rtw_disable_lps_deep_mode = true;
1766
1767 return 1;
1768 }
1769
1770 static const struct dmi_system_id rtw_pci_quirks[] = {
1771 {
1772 .callback = rtw_pci_disable_caps,
1773 .ident = "HP Notebook - P3S95EA#ACB",
1774 .matches = {
1775 DMI_MATCH(DMI_SYS_VENDOR, "HP"),
1776 DMI_MATCH(DMI_PRODUCT_NAME, "HP Notebook"),
1777 DMI_MATCH(DMI_PRODUCT_SKU, "P3S95EA#ACB"),
1778 },
1779 .driver_data = (void *)(BIT(QUIRK_DIS_CAP_PCI_ASPM) |
1780 BIT(QUIRK_DIS_CAP_LPS_DEEP)),
1781 },
1782 {
1783 .callback = rtw_pci_disable_caps,
1784 .ident = "ASUS TUF Gaming A15 FA506II",
1785 .matches = {
1786 DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
1787 DMI_MATCH(DMI_BOARD_NAME, "FA506II"),
1788 },
1789 .driver_data = (void *)(BIT(QUIRK_DIS_CAP_PCI_ASPM) |
1790 BIT(QUIRK_DIS_CAP_LPS_DEEP)),
1791 },
1792 {}
1793 };
1794
rtw_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)1795 int rtw_pci_probe(struct pci_dev *pdev,
1796 const struct pci_device_id *id)
1797 {
1798 struct pci_dev *bridge = pci_upstream_bridge(pdev);
1799 struct ieee80211_hw *hw;
1800 struct rtw_dev *rtwdev;
1801 struct rtw_pci *rtwpci;
1802 int drv_data_size;
1803 int ret;
1804
1805 drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci);
1806 hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops);
1807 if (!hw) {
1808 dev_err(&pdev->dev, "failed to allocate hw\n");
1809 return -ENOMEM;
1810 }
1811
1812 rtwdev = hw->priv;
1813 rtwdev->hw = hw;
1814 rtwdev->dev = &pdev->dev;
1815 rtwdev->chip = (struct rtw_chip_info *)id->driver_data;
1816 rtwdev->hci.ops = &rtw_pci_ops;
1817 rtwdev->hci.type = RTW_HCI_TYPE_PCIE;
1818
1819 rtwpci = (struct rtw_pci *)rtwdev->priv;
1820 atomic_set(&rtwpci->link_usage, 1);
1821
1822 dmi_check_system(rtw_pci_quirks);
1823
1824 ret = rtw_core_init(rtwdev);
1825 if (ret)
1826 goto err_release_hw;
1827
1828 rtw_dbg(rtwdev, RTW_DBG_PCI,
1829 "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n",
1830 pdev->vendor, pdev->device, pdev->revision);
1831
1832 ret = rtw_pci_claim(rtwdev, pdev);
1833 if (ret) {
1834 rtw_err(rtwdev, "failed to claim pci device\n");
1835 goto err_deinit_core;
1836 }
1837
1838 ret = rtw_pci_setup_resource(rtwdev, pdev);
1839 if (ret) {
1840 rtw_err(rtwdev, "failed to setup pci resources\n");
1841 goto err_pci_declaim;
1842 }
1843
1844 ret = rtw_pci_napi_init(rtwdev);
1845 if (ret) {
1846 rtw_err(rtwdev, "failed to setup NAPI\n");
1847 goto err_destroy_rsrc;
1848 }
1849
1850 ret = rtw_chip_info_setup(rtwdev);
1851 if (ret) {
1852 rtw_err(rtwdev, "failed to setup chip information\n");
1853 goto err_destroy_pci;
1854 }
1855
1856 /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */
1857 if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C &&
1858 bridge && bridge->vendor == PCI_VENDOR_ID_INTEL)
1859 rtwpci->rx_no_aspm = true;
1860
1861 rtw_pci_phy_cfg(rtwdev);
1862
1863 ret = rtw_register_hw(rtwdev, hw);
1864 if (ret) {
1865 rtw_err(rtwdev, "failed to register hw\n");
1866 goto err_destroy_pci;
1867 }
1868
1869 ret = rtw_pci_request_irq(rtwdev, pdev);
1870 if (ret) {
1871 ieee80211_unregister_hw(hw);
1872 goto err_destroy_pci;
1873 }
1874
1875 return 0;
1876
1877 err_destroy_pci:
1878 rtw_pci_napi_deinit(rtwdev);
1879
1880 err_destroy_rsrc:
1881 rtw_pci_destroy(rtwdev, pdev);
1882
1883 err_pci_declaim:
1884 rtw_pci_declaim(rtwdev, pdev);
1885
1886 err_deinit_core:
1887 rtw_core_deinit(rtwdev);
1888
1889 err_release_hw:
1890 ieee80211_free_hw(hw);
1891
1892 return ret;
1893 }
1894 EXPORT_SYMBOL(rtw_pci_probe);
1895
rtw_pci_remove(struct pci_dev * pdev)1896 void rtw_pci_remove(struct pci_dev *pdev)
1897 {
1898 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1899 struct rtw_dev *rtwdev;
1900 struct rtw_pci *rtwpci;
1901
1902 if (!hw)
1903 return;
1904
1905 rtwdev = hw->priv;
1906 rtwpci = (struct rtw_pci *)rtwdev->priv;
1907
1908 rtw_unregister_hw(rtwdev, hw);
1909 rtw_pci_disable_interrupt(rtwdev, rtwpci);
1910 rtw_pci_napi_deinit(rtwdev);
1911 rtw_pci_destroy(rtwdev, pdev);
1912 rtw_pci_declaim(rtwdev, pdev);
1913 rtw_pci_free_irq(rtwdev, pdev);
1914 rtw_core_deinit(rtwdev);
1915 ieee80211_free_hw(hw);
1916 }
1917 EXPORT_SYMBOL(rtw_pci_remove);
1918
rtw_pci_shutdown(struct pci_dev * pdev)1919 void rtw_pci_shutdown(struct pci_dev *pdev)
1920 {
1921 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1922 struct rtw_dev *rtwdev;
1923 const struct rtw_chip_info *chip;
1924
1925 if (!hw)
1926 return;
1927
1928 rtwdev = hw->priv;
1929 chip = rtwdev->chip;
1930
1931 if (chip->ops->shutdown)
1932 chip->ops->shutdown(rtwdev);
1933
1934 pci_set_power_state(pdev, PCI_D3hot);
1935 }
1936 EXPORT_SYMBOL(rtw_pci_shutdown);
1937
1938 MODULE_AUTHOR("Realtek Corporation");
1939 MODULE_DESCRIPTION("Realtek PCI 802.11ac wireless driver");
1940 MODULE_LICENSE("Dual BSD/GPL");
1941