xref: /freebsd/sys/contrib/dev/rtw88/pci.c (revision b077aed33b7b6aefca7b17ddb250cf521f938613)
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2018-2019  Realtek Corporation
3  */
4 
5 #if defined(__FreeBSD__)
6 #define	LINUXKPI_PARAM_PREFIX	rtw88_pci_
7 #endif
8 
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include "main.h"
12 #include "pci.h"
13 #include "reg.h"
14 #include "tx.h"
15 #include "rx.h"
16 #include "fw.h"
17 #include "ps.h"
18 #include "debug.h"
19 #if defined(__FreeBSD__)
20 #include <linux/pm.h>
21 #endif
22 
23 static bool rtw_disable_msi;
24 static bool rtw_pci_disable_aspm;
25 module_param_named(disable_msi, rtw_disable_msi, bool, 0644);
26 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644);
27 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support");
28 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support");
29 
30 static u32 rtw_pci_tx_queue_idx_addr[] = {
31 	[RTW_TX_QUEUE_BK]	= RTK_PCI_TXBD_IDX_BKQ,
32 	[RTW_TX_QUEUE_BE]	= RTK_PCI_TXBD_IDX_BEQ,
33 	[RTW_TX_QUEUE_VI]	= RTK_PCI_TXBD_IDX_VIQ,
34 	[RTW_TX_QUEUE_VO]	= RTK_PCI_TXBD_IDX_VOQ,
35 	[RTW_TX_QUEUE_MGMT]	= RTK_PCI_TXBD_IDX_MGMTQ,
36 	[RTW_TX_QUEUE_HI0]	= RTK_PCI_TXBD_IDX_HI0Q,
37 	[RTW_TX_QUEUE_H2C]	= RTK_PCI_TXBD_IDX_H2CQ,
38 };
39 
40 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, u8 queue)
41 {
42 	switch (queue) {
43 	case RTW_TX_QUEUE_BCN:
44 		return TX_DESC_QSEL_BEACON;
45 	case RTW_TX_QUEUE_H2C:
46 		return TX_DESC_QSEL_H2C;
47 	case RTW_TX_QUEUE_MGMT:
48 		return TX_DESC_QSEL_MGMT;
49 	case RTW_TX_QUEUE_HI0:
50 		return TX_DESC_QSEL_HIGH;
51 	default:
52 		return skb->priority;
53 	}
54 };
55 
56 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr)
57 {
58 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
59 
60 #if defined(__linux__)
61 	return readb(rtwpci->mmap + addr);
62 #elif defined(__FreeBSD__)
63 	u8 val;
64 
65 	val = bus_read_1((struct resource *)rtwpci->mmap, addr);
66 	rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val);
67 	return (val);
68 #endif
69 }
70 
71 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr)
72 {
73 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
74 
75 #if defined(__linux__)
76 	return readw(rtwpci->mmap + addr);
77 #elif defined(__FreeBSD__)
78 	u16 val;
79 
80 	val = bus_read_2((struct resource *)rtwpci->mmap, addr);
81 	rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val);
82 	return (val);
83 #endif
84 }
85 
86 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr)
87 {
88 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
89 
90 #if defined(__linux__)
91 	return readl(rtwpci->mmap + addr);
92 #elif defined(__FreeBSD__)
93 	u32 val;
94 
95 	val = bus_read_4((struct resource *)rtwpci->mmap, addr);
96 	rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
97 	return (val);
98 #endif
99 }
100 
101 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val)
102 {
103 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
104 
105 #if defined(__linux__)
106 	writeb(val, rtwpci->mmap + addr);
107 #elif defined(__FreeBSD__)
108 	rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, val);
109 	return (bus_write_1((struct resource *)rtwpci->mmap, addr, val));
110 #endif
111 }
112 
113 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val)
114 {
115 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
116 
117 #if defined(__linux__)
118 	writew(val, rtwpci->mmap + addr);
119 #elif defined(__FreeBSD__)
120 	rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, val);
121 	return (bus_write_2((struct resource *)rtwpci->mmap, addr, val));
122 #endif
123 }
124 
125 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
126 {
127 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
128 
129 #if defined(__linux__)
130 	writel(val, rtwpci->mmap + addr);
131 #elif defined(__FreeBSD__)
132 	rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, val);
133 	return (bus_write_4((struct resource *)rtwpci->mmap, addr, val));
134 #endif
135 }
136 
137 #if defined(__linux__) && 0
138 static inline void *rtw_pci_get_tx_desc(struct rtw_pci_tx_ring *tx_ring, u8 idx)
139 {
140 	int offset = tx_ring->r.desc_size * idx;
141 
142 	return tx_ring->r.head + offset;
143 }
144 #endif
145 
146 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev,
147 				      struct rtw_pci_tx_ring *tx_ring)
148 {
149 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
150 	struct rtw_pci_tx_data *tx_data;
151 	struct sk_buff *skb, *tmp;
152 	dma_addr_t dma;
153 
154 	/* free every skb remained in tx list */
155 	skb_queue_walk_safe(&tx_ring->queue, skb, tmp) {
156 		__skb_unlink(skb, &tx_ring->queue);
157 		tx_data = rtw_pci_get_tx_data(skb);
158 		dma = tx_data->dma;
159 
160 		dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE);
161 		dev_kfree_skb_any(skb);
162 	}
163 }
164 
165 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev,
166 				 struct rtw_pci_tx_ring *tx_ring)
167 {
168 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
169 	u8 *head = tx_ring->r.head;
170 	u32 len = tx_ring->r.len;
171 	int ring_sz = len * tx_ring->r.desc_size;
172 
173 	rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
174 
175 	/* free the ring itself */
176 	dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma);
177 	tx_ring->r.head = NULL;
178 }
179 
180 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev,
181 				      struct rtw_pci_rx_ring *rx_ring)
182 {
183 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
184 	struct sk_buff *skb;
185 	int buf_sz = RTK_PCI_RX_BUF_SIZE;
186 	dma_addr_t dma;
187 	int i;
188 
189 	for (i = 0; i < rx_ring->r.len; i++) {
190 		skb = rx_ring->buf[i];
191 		if (!skb)
192 			continue;
193 
194 		dma = *((dma_addr_t *)skb->cb);
195 		dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
196 		dev_kfree_skb(skb);
197 		rx_ring->buf[i] = NULL;
198 	}
199 }
200 
201 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev,
202 				 struct rtw_pci_rx_ring *rx_ring)
203 {
204 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
205 	u8 *head = rx_ring->r.head;
206 	int ring_sz = rx_ring->r.desc_size * rx_ring->r.len;
207 
208 	rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring);
209 
210 	dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma);
211 }
212 
213 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev)
214 {
215 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
216 	struct rtw_pci_tx_ring *tx_ring;
217 	struct rtw_pci_rx_ring *rx_ring;
218 	int i;
219 
220 	for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
221 		tx_ring = &rtwpci->tx_rings[i];
222 		rtw_pci_free_tx_ring(rtwdev, tx_ring);
223 	}
224 
225 	for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) {
226 		rx_ring = &rtwpci->rx_rings[i];
227 		rtw_pci_free_rx_ring(rtwdev, rx_ring);
228 	}
229 }
230 
231 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev,
232 				struct rtw_pci_tx_ring *tx_ring,
233 				u8 desc_size, u32 len)
234 {
235 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
236 	int ring_sz = desc_size * len;
237 	dma_addr_t dma;
238 	u8 *head;
239 
240 	if (len > TRX_BD_IDX_MASK) {
241 		rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len);
242 		return -EINVAL;
243 	}
244 
245 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
246 	if (!head) {
247 		rtw_err(rtwdev, "failed to allocate tx ring\n");
248 		return -ENOMEM;
249 	}
250 
251 	skb_queue_head_init(&tx_ring->queue);
252 	tx_ring->r.head = head;
253 	tx_ring->r.dma = dma;
254 	tx_ring->r.len = len;
255 	tx_ring->r.desc_size = desc_size;
256 	tx_ring->r.wp = 0;
257 	tx_ring->r.rp = 0;
258 
259 	return 0;
260 }
261 
262 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb,
263 				 struct rtw_pci_rx_ring *rx_ring,
264 				 u32 idx, u32 desc_sz)
265 {
266 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
267 	struct rtw_pci_rx_buffer_desc *buf_desc;
268 	int buf_sz = RTK_PCI_RX_BUF_SIZE;
269 	dma_addr_t dma;
270 
271 	if (!skb)
272 		return -EINVAL;
273 
274 	dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
275 	if (dma_mapping_error(&pdev->dev, dma))
276 		return -EBUSY;
277 
278 	*((dma_addr_t *)skb->cb) = dma;
279 	buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
280 						     idx * desc_sz);
281 	memset(buf_desc, 0, sizeof(*buf_desc));
282 	buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
283 	buf_desc->dma = cpu_to_le32(dma);
284 
285 	return 0;
286 }
287 
288 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma,
289 					struct rtw_pci_rx_ring *rx_ring,
290 					u32 idx, u32 desc_sz)
291 {
292 	struct device *dev = rtwdev->dev;
293 	struct rtw_pci_rx_buffer_desc *buf_desc;
294 	int buf_sz = RTK_PCI_RX_BUF_SIZE;
295 
296 	dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE);
297 
298 	buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
299 						     idx * desc_sz);
300 	memset(buf_desc, 0, sizeof(*buf_desc));
301 	buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
302 	buf_desc->dma = cpu_to_le32(dma);
303 }
304 
305 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev,
306 				struct rtw_pci_rx_ring *rx_ring,
307 				u8 desc_size, u32 len)
308 {
309 	struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
310 	struct sk_buff *skb = NULL;
311 	dma_addr_t dma;
312 	u8 *head;
313 	int ring_sz = desc_size * len;
314 	int buf_sz = RTK_PCI_RX_BUF_SIZE;
315 	int i, allocated;
316 	int ret = 0;
317 
318 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
319 	if (!head) {
320 		rtw_err(rtwdev, "failed to allocate rx ring\n");
321 		return -ENOMEM;
322 	}
323 	rx_ring->r.head = head;
324 
325 	for (i = 0; i < len; i++) {
326 		skb = dev_alloc_skb(buf_sz);
327 		if (!skb) {
328 			allocated = i;
329 			ret = -ENOMEM;
330 			goto err_out;
331 		}
332 
333 		memset(skb->data, 0, buf_sz);
334 		rx_ring->buf[i] = skb;
335 		ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size);
336 		if (ret) {
337 			allocated = i;
338 			dev_kfree_skb_any(skb);
339 			goto err_out;
340 		}
341 	}
342 
343 	rx_ring->r.dma = dma;
344 	rx_ring->r.len = len;
345 	rx_ring->r.desc_size = desc_size;
346 	rx_ring->r.wp = 0;
347 	rx_ring->r.rp = 0;
348 
349 	return 0;
350 
351 err_out:
352 	for (i = 0; i < allocated; i++) {
353 		skb = rx_ring->buf[i];
354 		if (!skb)
355 			continue;
356 		dma = *((dma_addr_t *)skb->cb);
357 		dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
358 		dev_kfree_skb_any(skb);
359 		rx_ring->buf[i] = NULL;
360 	}
361 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
362 
363 	rtw_err(rtwdev, "failed to init rx buffer\n");
364 
365 	return ret;
366 }
367 
368 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev)
369 {
370 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
371 	struct rtw_pci_tx_ring *tx_ring;
372 	struct rtw_pci_rx_ring *rx_ring;
373 	struct rtw_chip_info *chip = rtwdev->chip;
374 	int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0;
375 	int tx_desc_size, rx_desc_size;
376 	u32 len;
377 	int ret;
378 
379 	tx_desc_size = chip->tx_buf_desc_sz;
380 
381 	for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
382 		tx_ring = &rtwpci->tx_rings[i];
383 		len = max_num_of_tx_queue(i);
384 		ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len);
385 		if (ret)
386 			goto out;
387 	}
388 
389 	rx_desc_size = chip->rx_buf_desc_sz;
390 
391 	for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) {
392 		rx_ring = &rtwpci->rx_rings[j];
393 		ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size,
394 					   RTK_MAX_RX_DESC_NUM);
395 		if (ret)
396 			goto out;
397 	}
398 
399 	return 0;
400 
401 out:
402 	tx_alloced = i;
403 	for (i = 0; i < tx_alloced; i++) {
404 		tx_ring = &rtwpci->tx_rings[i];
405 		rtw_pci_free_tx_ring(rtwdev, tx_ring);
406 	}
407 
408 	rx_alloced = j;
409 	for (j = 0; j < rx_alloced; j++) {
410 		rx_ring = &rtwpci->rx_rings[j];
411 		rtw_pci_free_rx_ring(rtwdev, rx_ring);
412 	}
413 
414 	return ret;
415 }
416 
417 static void rtw_pci_deinit(struct rtw_dev *rtwdev)
418 {
419 	rtw_pci_free_trx_ring(rtwdev);
420 }
421 
422 static int rtw_pci_init(struct rtw_dev *rtwdev)
423 {
424 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
425 	int ret = 0;
426 
427 	rtwpci->irq_mask[0] = IMR_HIGHDOK |
428 			      IMR_MGNTDOK |
429 			      IMR_BKDOK |
430 			      IMR_BEDOK |
431 			      IMR_VIDOK |
432 			      IMR_VODOK |
433 			      IMR_ROK |
434 			      IMR_BCNDMAINT_E |
435 			      IMR_C2HCMD |
436 			      0;
437 	rtwpci->irq_mask[1] = IMR_TXFOVW |
438 			      0;
439 	rtwpci->irq_mask[3] = IMR_H2CDOK |
440 			      0;
441 	spin_lock_init(&rtwpci->irq_lock);
442 	spin_lock_init(&rtwpci->hwirq_lock);
443 	ret = rtw_pci_init_trx_ring(rtwdev);
444 
445 	return ret;
446 }
447 
448 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)
449 {
450 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
451 	u32 len;
452 	u8 tmp;
453 	dma_addr_t dma;
454 
455 	tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3);
456 	rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7);
457 
458 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;
459 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma);
460 
461 	if (!rtw_chip_wcpu_11n(rtwdev)) {
462 		len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;
463 		dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;
464 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;
465 		rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;
466 		rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK);
467 		rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma);
468 	}
469 
470 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;
471 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;
472 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;
473 	rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;
474 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK);
475 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma);
476 
477 	len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;
478 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;
479 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;
480 	rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;
481 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK);
482 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma);
483 
484 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;
485 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;
486 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;
487 	rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;
488 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK);
489 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma);
490 
491 	len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;
492 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;
493 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;
494 	rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;
495 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK);
496 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma);
497 
498 	len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;
499 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;
500 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;
501 	rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;
502 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK);
503 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma);
504 
505 	len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;
506 	dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;
507 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;
508 	rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;
509 	rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK);
510 	rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma);
511 
512 	len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;
513 	dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;
514 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;
515 	rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;
516 	rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK);
517 	rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma);
518 
519 	/* reset read/write point */
520 	rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff);
521 
522 	/* reset H2C Queue index in a single write */
523 	if (rtw_chip_wcpu_11ac(rtwdev))
524 		rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR,
525 				BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX);
526 }
527 
528 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev)
529 {
530 	rtw_pci_reset_buf_desc(rtwdev);
531 }
532 
533 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev,
534 				     struct rtw_pci *rtwpci, bool exclude_rx)
535 {
536 	unsigned long flags;
537 	u32 imr0_unmask = exclude_rx ? IMR_ROK : 0;
538 
539 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
540 
541 	rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);
542 	rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);
543 	if (rtw_chip_wcpu_11ac(rtwdev))
544 		rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);
545 
546 	rtwpci->irq_enabled = true;
547 
548 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
549 }
550 
551 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev,
552 				      struct rtw_pci *rtwpci)
553 {
554 	unsigned long flags;
555 
556 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
557 
558 	if (!rtwpci->irq_enabled)
559 		goto out;
560 
561 	rtw_write32(rtwdev, RTK_PCI_HIMR0, 0);
562 	rtw_write32(rtwdev, RTK_PCI_HIMR1, 0);
563 	if (rtw_chip_wcpu_11ac(rtwdev))
564 		rtw_write32(rtwdev, RTK_PCI_HIMR3, 0);
565 
566 	rtwpci->irq_enabled = false;
567 
568 out:
569 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
570 }
571 
572 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
573 {
574 	/* reset dma and rx tag */
575 	rtw_write32_set(rtwdev, RTK_PCI_CTRL,
576 			BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN);
577 	rtwpci->rx_tag = 0;
578 }
579 
580 static int rtw_pci_setup(struct rtw_dev *rtwdev)
581 {
582 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
583 
584 	rtw_pci_reset_trx_ring(rtwdev);
585 	rtw_pci_dma_reset(rtwdev, rtwpci);
586 
587 	return 0;
588 }
589 
590 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
591 {
592 	struct rtw_pci_tx_ring *tx_ring;
593 	u8 queue;
594 
595 	rtw_pci_reset_trx_ring(rtwdev);
596 	for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
597 		tx_ring = &rtwpci->tx_rings[queue];
598 		rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
599 	}
600 }
601 
602 static void rtw_pci_napi_start(struct rtw_dev *rtwdev)
603 {
604 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
605 
606 	if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
607 		return;
608 
609 	napi_enable(&rtwpci->napi);
610 }
611 
612 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev)
613 {
614 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
615 
616 	if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
617 		return;
618 
619 	napi_synchronize(&rtwpci->napi);
620 	napi_disable(&rtwpci->napi);
621 }
622 
623 static int rtw_pci_start(struct rtw_dev *rtwdev)
624 {
625 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
626 
627 	rtw_pci_napi_start(rtwdev);
628 
629 	spin_lock_bh(&rtwpci->irq_lock);
630 	rtwpci->running = true;
631 	rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
632 	spin_unlock_bh(&rtwpci->irq_lock);
633 
634 	return 0;
635 }
636 
637 static void rtw_pci_stop(struct rtw_dev *rtwdev)
638 {
639 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
640 	struct pci_dev *pdev = rtwpci->pdev;
641 
642 	spin_lock_bh(&rtwpci->irq_lock);
643 	rtwpci->running = false;
644 	rtw_pci_disable_interrupt(rtwdev, rtwpci);
645 	spin_unlock_bh(&rtwpci->irq_lock);
646 
647 	synchronize_irq(pdev->irq);
648 	rtw_pci_napi_stop(rtwdev);
649 
650 	spin_lock_bh(&rtwpci->irq_lock);
651 	rtw_pci_dma_release(rtwdev, rtwpci);
652 	spin_unlock_bh(&rtwpci->irq_lock);
653 }
654 
655 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev)
656 {
657 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
658 	struct rtw_pci_tx_ring *tx_ring;
659 	bool tx_empty = true;
660 	u8 queue;
661 
662 	if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
663 		goto enter_deep_ps;
664 
665 	lockdep_assert_held(&rtwpci->irq_lock);
666 
667 	/* Deep PS state is not allowed to TX-DMA */
668 	for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
669 		/* BCN queue is rsvd page, does not have DMA interrupt
670 		 * H2C queue is managed by firmware
671 		 */
672 		if (queue == RTW_TX_QUEUE_BCN ||
673 		    queue == RTW_TX_QUEUE_H2C)
674 			continue;
675 
676 		tx_ring = &rtwpci->tx_rings[queue];
677 
678 		/* check if there is any skb DMAing */
679 		if (skb_queue_len(&tx_ring->queue)) {
680 			tx_empty = false;
681 			break;
682 		}
683 	}
684 
685 	if (!tx_empty) {
686 		rtw_dbg(rtwdev, RTW_DBG_PS,
687 			"TX path not empty, cannot enter deep power save state\n");
688 		return;
689 	}
690 enter_deep_ps:
691 	set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags);
692 	rtw_power_mode_change(rtwdev, true);
693 }
694 
695 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev)
696 {
697 #if defined(__linux__)
698 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
699 
700 	lockdep_assert_held(&rtwpci->irq_lock);
701 #elif defined(__FreeBSD__)
702 	lockdep_assert_held(&((struct rtw_pci *)rtwdev->priv)->irq_lock);
703 #endif
704 
705 	if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
706 		rtw_power_mode_change(rtwdev, false);
707 }
708 
709 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter)
710 {
711 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
712 
713 	spin_lock_bh(&rtwpci->irq_lock);
714 
715 	if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
716 		rtw_pci_deep_ps_enter(rtwdev);
717 
718 	if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
719 		rtw_pci_deep_ps_leave(rtwdev);
720 
721 	spin_unlock_bh(&rtwpci->irq_lock);
722 }
723 
724 static u8 ac_to_hwq[] = {
725 	[IEEE80211_AC_VO] = RTW_TX_QUEUE_VO,
726 	[IEEE80211_AC_VI] = RTW_TX_QUEUE_VI,
727 	[IEEE80211_AC_BE] = RTW_TX_QUEUE_BE,
728 	[IEEE80211_AC_BK] = RTW_TX_QUEUE_BK,
729 };
730 
731 #if defined(__linux__)
732 static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS);
733 #elif defined(__FreeBSD__)
734 rtw88_static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS);
735 #endif
736 
737 static u8 rtw_hw_queue_mapping(struct sk_buff *skb)
738 {
739 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
740 	__le16 fc = hdr->frame_control;
741 	u8 q_mapping = skb_get_queue_mapping(skb);
742 	u8 queue;
743 
744 	if (unlikely(ieee80211_is_beacon(fc)))
745 		queue = RTW_TX_QUEUE_BCN;
746 	else if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc)))
747 		queue = RTW_TX_QUEUE_MGMT;
748 	else if (is_broadcast_ether_addr(hdr->addr1) ||
749 		 is_multicast_ether_addr(hdr->addr1))
750 		queue = RTW_TX_QUEUE_HI0;
751 	else if (WARN_ON_ONCE(q_mapping >= ARRAY_SIZE(ac_to_hwq)))
752 		queue = ac_to_hwq[IEEE80211_AC_BE];
753 	else
754 		queue = ac_to_hwq[q_mapping];
755 
756 	return queue;
757 }
758 
759 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,
760 				      struct rtw_pci_tx_ring *ring)
761 {
762 	struct sk_buff *prev = skb_dequeue(&ring->queue);
763 	struct rtw_pci_tx_data *tx_data;
764 	dma_addr_t dma;
765 
766 	if (!prev)
767 		return;
768 
769 	tx_data = rtw_pci_get_tx_data(prev);
770 	dma = tx_data->dma;
771 	dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);
772 	dev_kfree_skb_any(prev);
773 }
774 
775 static void rtw_pci_dma_check(struct rtw_dev *rtwdev,
776 			      struct rtw_pci_rx_ring *rx_ring,
777 			      u32 idx)
778 {
779 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
780 	struct rtw_chip_info *chip = rtwdev->chip;
781 	struct rtw_pci_rx_buffer_desc *buf_desc;
782 	u32 desc_sz = chip->rx_buf_desc_sz;
783 	u16 total_pkt_size;
784 
785 	buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
786 						     idx * desc_sz);
787 	total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size);
788 
789 	/* rx tag mismatch, throw a warning */
790 	if (total_pkt_size != rtwpci->rx_tag)
791 		rtw_warn(rtwdev, "pci bus timeout, check dma status\n");
792 
793 	rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;
794 }
795 
796 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q)
797 {
798 	u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q];
799 	u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2);
800 
801 	return FIELD_GET(TRX_BD_IDX_MASK, bd_idx);
802 }
803 
804 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop)
805 {
806 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
807 	struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];
808 	u32 cur_rp;
809 	u8 i;
810 
811 	/* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a
812 	 * bit dynamic, it's hard to define a reasonable fixed total timeout to
813 	 * use read_poll_timeout* helper. Instead, we can ensure a reasonable
814 	 * polling times, so we just use for loop with udelay here.
815 	 */
816 	for (i = 0; i < 30; i++) {
817 		cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q);
818 		if (cur_rp == ring->r.wp)
819 			return;
820 
821 		udelay(1);
822 	}
823 
824 	if (!drop)
825 		rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q);
826 }
827 
828 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues,
829 				   bool drop)
830 {
831 	u8 q;
832 
833 	for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) {
834 		/* It may be not necessary to flush BCN and H2C tx queues. */
835 		if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C)
836 			continue;
837 
838 		if (pci_queues & BIT(q))
839 			__pci_flush_queue(rtwdev, q, drop);
840 	}
841 }
842 
843 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop)
844 {
845 	u32 pci_queues = 0;
846 	u8 i;
847 
848 	/* If all of the hardware queues are requested to flush,
849 	 * flush all of the pci queues.
850 	 */
851 	if (queues == BIT(rtwdev->hw->queues) - 1) {
852 		pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1;
853 	} else {
854 		for (i = 0; i < rtwdev->hw->queues; i++)
855 			if (queues & BIT(i))
856 				pci_queues |= BIT(ac_to_hwq[i]);
857 	}
858 
859 	__rtw_pci_flush_queues(rtwdev, pci_queues, drop);
860 }
861 
862 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, u8 queue)
863 {
864 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
865 	struct rtw_pci_tx_ring *ring;
866 	u32 bd_idx;
867 
868 	ring = &rtwpci->tx_rings[queue];
869 	bd_idx = rtw_pci_tx_queue_idx_addr[queue];
870 
871 	spin_lock_bh(&rtwpci->irq_lock);
872 	if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
873 		rtw_pci_deep_ps_leave(rtwdev);
874 	rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK);
875 	spin_unlock_bh(&rtwpci->irq_lock);
876 }
877 
878 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev)
879 {
880 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
881 	u8 queue;
882 
883 	for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++)
884 		if (test_and_clear_bit(queue, rtwpci->tx_queued))
885 			rtw_pci_tx_kick_off_queue(rtwdev, queue);
886 }
887 
888 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev,
889 				 struct rtw_tx_pkt_info *pkt_info,
890 				 struct sk_buff *skb, u8 queue)
891 {
892 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
893 	struct rtw_chip_info *chip = rtwdev->chip;
894 	struct rtw_pci_tx_ring *ring;
895 	struct rtw_pci_tx_data *tx_data;
896 	dma_addr_t dma;
897 	u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz;
898 	u32 tx_buf_desc_sz = chip->tx_buf_desc_sz;
899 	u32 size;
900 	u32 psb_len;
901 	u8 *pkt_desc;
902 	struct rtw_pci_tx_buffer_desc *buf_desc;
903 
904 	ring = &rtwpci->tx_rings[queue];
905 
906 	size = skb->len;
907 
908 	if (queue == RTW_TX_QUEUE_BCN)
909 		rtw_pci_release_rsvd_page(rtwpci, ring);
910 	else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len))
911 		return -ENOSPC;
912 
913 	pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz);
914 	memset(pkt_desc, 0, tx_pkt_desc_sz);
915 	pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue);
916 	rtw_tx_fill_tx_desc(pkt_info, skb);
917 	dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,
918 			     DMA_TO_DEVICE);
919 	if (dma_mapping_error(&rtwpci->pdev->dev, dma))
920 		return -EBUSY;
921 
922 	/* after this we got dma mapped, there is no way back */
923 	buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz);
924 	memset(buf_desc, 0, tx_buf_desc_sz);
925 	psb_len = (skb->len - 1) / 128 + 1;
926 	if (queue == RTW_TX_QUEUE_BCN)
927 		psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET;
928 
929 	buf_desc[0].psb_len = cpu_to_le16(psb_len);
930 	buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz);
931 	buf_desc[0].dma = cpu_to_le32(dma);
932 	buf_desc[1].buf_size = cpu_to_le16(size);
933 	buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz);
934 
935 	tx_data = rtw_pci_get_tx_data(skb);
936 	tx_data->dma = dma;
937 	tx_data->sn = pkt_info->sn;
938 
939 	spin_lock_bh(&rtwpci->irq_lock);
940 
941 	skb_queue_tail(&ring->queue, skb);
942 
943 	if (queue == RTW_TX_QUEUE_BCN)
944 		goto out_unlock;
945 
946 	/* update write-index, and kick it off later */
947 	set_bit(queue, rtwpci->tx_queued);
948 	if (++ring->r.wp >= ring->r.len)
949 		ring->r.wp = 0;
950 
951 out_unlock:
952 	spin_unlock_bh(&rtwpci->irq_lock);
953 
954 	return 0;
955 }
956 
957 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf,
958 					u32 size)
959 {
960 	struct sk_buff *skb;
961 	struct rtw_tx_pkt_info pkt_info = {0};
962 	u8 reg_bcn_work;
963 	int ret;
964 
965 	skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size);
966 	if (!skb)
967 		return -ENOMEM;
968 
969 	ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN);
970 	if (ret) {
971 #if defined(__FreeBSD__)
972 		dev_kfree_skb_any(skb);
973 #endif
974 		rtw_err(rtwdev, "failed to write rsvd page data\n");
975 		return ret;
976 	}
977 
978 	/* reserved pages go through beacon queue */
979 	reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK);
980 	reg_bcn_work |= BIT_PCI_BCNQ_FLAG;
981 	rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work);
982 
983 	return 0;
984 }
985 
986 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size)
987 {
988 	struct sk_buff *skb;
989 	struct rtw_tx_pkt_info pkt_info = {0};
990 	int ret;
991 
992 	skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size);
993 	if (!skb)
994 		return -ENOMEM;
995 
996 	ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C);
997 	if (ret) {
998 #if defined(__FreeBSD__)
999 		dev_kfree_skb_any(skb);
1000 #endif
1001 		rtw_err(rtwdev, "failed to write h2c data\n");
1002 		return ret;
1003 	}
1004 
1005 	rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C);
1006 
1007 	return 0;
1008 }
1009 
1010 static int rtw_pci_tx_write(struct rtw_dev *rtwdev,
1011 			    struct rtw_tx_pkt_info *pkt_info,
1012 			    struct sk_buff *skb)
1013 {
1014 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1015 	struct rtw_pci_tx_ring *ring;
1016 	u8 queue = rtw_hw_queue_mapping(skb);
1017 	int ret;
1018 
1019 	ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue);
1020 	if (ret)
1021 		return ret;
1022 
1023 	ring = &rtwpci->tx_rings[queue];
1024 	spin_lock_bh(&rtwpci->irq_lock);
1025 	if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) {
1026 		ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb));
1027 		ring->queue_stopped = true;
1028 	}
1029 	spin_unlock_bh(&rtwpci->irq_lock);
1030 
1031 	return 0;
1032 }
1033 
1034 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
1035 			   u8 hw_queue)
1036 {
1037 	struct ieee80211_hw *hw = rtwdev->hw;
1038 	struct ieee80211_tx_info *info;
1039 	struct rtw_pci_tx_ring *ring;
1040 	struct rtw_pci_tx_data *tx_data;
1041 	struct sk_buff *skb;
1042 	u32 count;
1043 	u32 bd_idx_addr;
1044 	u32 bd_idx, cur_rp, rp_idx;
1045 	u16 q_map;
1046 
1047 	ring = &rtwpci->tx_rings[hw_queue];
1048 
1049 	bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue];
1050 	bd_idx = rtw_read32(rtwdev, bd_idx_addr);
1051 	cur_rp = bd_idx >> 16;
1052 	cur_rp &= TRX_BD_IDX_MASK;
1053 	rp_idx = ring->r.rp;
1054 	if (cur_rp >= ring->r.rp)
1055 		count = cur_rp - ring->r.rp;
1056 	else
1057 		count = ring->r.len - (ring->r.rp - cur_rp);
1058 
1059 	while (count--) {
1060 		skb = skb_dequeue(&ring->queue);
1061 		if (!skb) {
1062 			rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n",
1063 				count, hw_queue, bd_idx, ring->r.rp, cur_rp);
1064 			break;
1065 		}
1066 		tx_data = rtw_pci_get_tx_data(skb);
1067 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
1068 				 DMA_TO_DEVICE);
1069 
1070 		/* just free command packets from host to card */
1071 		if (hw_queue == RTW_TX_QUEUE_H2C) {
1072 			dev_kfree_skb_irq(skb);
1073 			continue;
1074 		}
1075 
1076 		if (ring->queue_stopped &&
1077 		    avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) {
1078 			q_map = skb_get_queue_mapping(skb);
1079 			ieee80211_wake_queue(hw, q_map);
1080 			ring->queue_stopped = false;
1081 		}
1082 
1083 		if (++rp_idx >= ring->r.len)
1084 			rp_idx = 0;
1085 
1086 		skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz);
1087 
1088 		info = IEEE80211_SKB_CB(skb);
1089 
1090 		/* enqueue to wait for tx report */
1091 		if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
1092 			rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn);
1093 			continue;
1094 		}
1095 
1096 		/* always ACK for others, then they won't be marked as drop */
1097 		if (info->flags & IEEE80211_TX_CTL_NO_ACK)
1098 			info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
1099 		else
1100 			info->flags |= IEEE80211_TX_STAT_ACK;
1101 
1102 		ieee80211_tx_info_clear_status(info);
1103 		ieee80211_tx_status_irqsafe(hw, skb);
1104 	}
1105 
1106 	ring->r.rp = cur_rp;
1107 }
1108 
1109 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev)
1110 {
1111 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1112 	struct napi_struct *napi = &rtwpci->napi;
1113 
1114 	napi_schedule(napi);
1115 }
1116 
1117 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev,
1118 				     struct rtw_pci *rtwpci)
1119 {
1120 	struct rtw_pci_rx_ring *ring;
1121 	int count = 0;
1122 	u32 tmp, cur_wp;
1123 
1124 	ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1125 	tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ);
1126 	cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK);
1127 	if (cur_wp >= ring->r.wp)
1128 		count = cur_wp - ring->r.wp;
1129 	else
1130 		count = ring->r.len - (ring->r.wp - cur_wp);
1131 
1132 	return count;
1133 }
1134 
1135 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
1136 			   u8 hw_queue, u32 limit)
1137 {
1138 	struct rtw_chip_info *chip = rtwdev->chip;
1139 	struct napi_struct *napi = &rtwpci->napi;
1140 	struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1141 	struct rtw_rx_pkt_stat pkt_stat;
1142 	struct ieee80211_rx_status rx_status;
1143 	struct sk_buff *skb, *new;
1144 	u32 cur_rp = ring->r.rp;
1145 	u32 count, rx_done = 0;
1146 	u32 pkt_offset;
1147 	u32 pkt_desc_sz = chip->rx_pkt_desc_sz;
1148 	u32 buf_desc_sz = chip->rx_buf_desc_sz;
1149 	u32 new_len;
1150 	u8 *rx_desc;
1151 	dma_addr_t dma;
1152 
1153 	count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);
1154 	count = min(count, limit);
1155 
1156 	while (count--) {
1157 		rtw_pci_dma_check(rtwdev, ring, cur_rp);
1158 		skb = ring->buf[cur_rp];
1159 		dma = *((dma_addr_t *)skb->cb);
1160 		dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE,
1161 					DMA_FROM_DEVICE);
1162 		rx_desc = skb->data;
1163 		chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status);
1164 
1165 		/* offset from rx_desc to payload */
1166 		pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz +
1167 			     pkt_stat.shift;
1168 
1169 		/* allocate a new skb for this frame,
1170 		 * discard the frame if none available
1171 		 */
1172 		new_len = pkt_stat.pkt_len + pkt_offset;
1173 		new = dev_alloc_skb(new_len);
1174 		if (WARN_ONCE(!new, "rx routine starvation\n"))
1175 			goto next_rp;
1176 
1177 		/* put the DMA data including rx_desc from phy to new skb */
1178 		skb_put_data(new, skb->data, new_len);
1179 
1180 		if (pkt_stat.is_c2h) {
1181 			rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new);
1182 		} else {
1183 			/* remove rx_desc */
1184 			skb_pull(new, pkt_offset);
1185 
1186 			rtw_rx_stats(rtwdev, pkt_stat.vif, new);
1187 			memcpy(new->cb, &rx_status, sizeof(rx_status));
1188 			ieee80211_rx_napi(rtwdev->hw, NULL, new, napi);
1189 			rx_done++;
1190 		}
1191 
1192 next_rp:
1193 		/* new skb delivered to mac80211, re-enable original skb DMA */
1194 		rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp,
1195 					    buf_desc_sz);
1196 
1197 		/* host read next element in ring */
1198 		if (++cur_rp >= ring->r.len)
1199 			cur_rp = 0;
1200 	}
1201 
1202 	ring->r.rp = cur_rp;
1203 	/* 'rp', the last position we have read, is seen as previous posistion
1204 	 * of 'wp' that is used to calculate 'count' next time.
1205 	 */
1206 	ring->r.wp = cur_rp;
1207 	rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp);
1208 
1209 	return rx_done;
1210 }
1211 
1212 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,
1213 				   struct rtw_pci *rtwpci, u32 *irq_status)
1214 {
1215 	unsigned long flags;
1216 
1217 	spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
1218 
1219 	irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0);
1220 	irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1);
1221 	if (rtw_chip_wcpu_11ac(rtwdev))
1222 		irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3);
1223 	else
1224 		irq_status[3] = 0;
1225 	irq_status[0] &= rtwpci->irq_mask[0];
1226 	irq_status[1] &= rtwpci->irq_mask[1];
1227 	irq_status[3] &= rtwpci->irq_mask[3];
1228 	rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]);
1229 	rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]);
1230 	if (rtw_chip_wcpu_11ac(rtwdev))
1231 		rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]);
1232 
1233 	spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
1234 }
1235 
1236 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev)
1237 {
1238 	struct rtw_dev *rtwdev = dev;
1239 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1240 
1241 	/* disable RTW PCI interrupt to avoid more interrupts before the end of
1242 	 * thread function
1243 	 *
1244 	 * disable HIMR here to also avoid new HISR flag being raised before
1245 	 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs
1246 	 * are cleared, the edge-triggered interrupt will not be generated when
1247 	 * a new HISR flag is set.
1248 	 */
1249 	rtw_pci_disable_interrupt(rtwdev, rtwpci);
1250 
1251 	return IRQ_WAKE_THREAD;
1252 }
1253 
1254 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev)
1255 {
1256 	struct rtw_dev *rtwdev = dev;
1257 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1258 	u32 irq_status[4];
1259 	bool rx = false;
1260 
1261 	spin_lock_bh(&rtwpci->irq_lock);
1262 	rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);
1263 
1264 	if (irq_status[0] & IMR_MGNTDOK)
1265 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);
1266 	if (irq_status[0] & IMR_HIGHDOK)
1267 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);
1268 	if (irq_status[0] & IMR_BEDOK)
1269 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);
1270 	if (irq_status[0] & IMR_BKDOK)
1271 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);
1272 	if (irq_status[0] & IMR_VODOK)
1273 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);
1274 	if (irq_status[0] & IMR_VIDOK)
1275 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);
1276 	if (irq_status[3] & IMR_H2CDOK)
1277 		rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);
1278 	if (irq_status[0] & IMR_ROK) {
1279 		rtw_pci_rx_isr(rtwdev);
1280 		rx = true;
1281 	}
1282 	if (unlikely(irq_status[0] & IMR_C2HCMD))
1283 		rtw_fw_c2h_cmd_isr(rtwdev);
1284 
1285 	/* all of the jobs for this interrupt have been done */
1286 	if (rtwpci->running)
1287 		rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);
1288 	spin_unlock_bh(&rtwpci->irq_lock);
1289 
1290 	return IRQ_HANDLED;
1291 }
1292 
1293 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev,
1294 			      struct pci_dev *pdev)
1295 {
1296 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1297 	unsigned long len;
1298 	u8 bar_id = 2;
1299 	int ret;
1300 
1301 	ret = pci_request_regions(pdev, KBUILD_MODNAME);
1302 	if (ret) {
1303 		rtw_err(rtwdev, "failed to request pci regions\n");
1304 		return ret;
1305 	}
1306 
1307 #if defined(__FreeBSD__)
1308 	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1309 	if (ret) {
1310 		rtw_err(rtwdev, "failed to set dma mask to 32-bit\n");
1311 		goto err_release_regions;
1312 	}
1313 
1314 	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1315 	if (ret) {
1316 		rtw_err(rtwdev, "failed to set consistent dma mask to 32-bit\n");
1317 		goto err_release_regions;
1318 	}
1319 #endif
1320 
1321 	len = pci_resource_len(pdev, bar_id);
1322 #if defined(__FreeBSD__)
1323 	linuxkpi_pcim_want_to_use_bus_functions(pdev);
1324 #endif
1325 	rtwpci->mmap = pci_iomap(pdev, bar_id, len);
1326 	if (!rtwpci->mmap) {
1327 		pci_release_regions(pdev);
1328 		rtw_err(rtwdev, "failed to map pci memory\n");
1329 		return -ENOMEM;
1330 	}
1331 
1332 	return 0;
1333 #if defined(__FreeBSD__)
1334 err_release_regions:
1335 	pci_release_regions(pdev);
1336 	return ret;
1337 #endif
1338 }
1339 
1340 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev,
1341 				 struct pci_dev *pdev)
1342 {
1343 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1344 
1345 	if (rtwpci->mmap) {
1346 		pci_iounmap(pdev, rtwpci->mmap);
1347 		pci_release_regions(pdev);
1348 	}
1349 }
1350 
1351 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data)
1352 {
1353 	u16 write_addr;
1354 	u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK);
1355 	u8 flag;
1356 	u8 cnt;
1357 
1358 	write_addr = addr & BITS_DBI_ADDR_MASK;
1359 	write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN);
1360 	rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data);
1361 	rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr);
1362 	rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16);
1363 
1364 	for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1365 		flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1366 		if (flag == 0)
1367 			return;
1368 
1369 		udelay(10);
1370 	}
1371 
1372 	WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr);
1373 }
1374 
1375 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value)
1376 {
1377 	u16 read_addr = addr & BITS_DBI_ADDR_MASK;
1378 	u8 flag;
1379 	u8 cnt;
1380 
1381 	rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr);
1382 	rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16);
1383 
1384 	for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1385 		flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1386 		if (flag == 0) {
1387 			read_addr = REG_DBI_RDATA_V1 + (addr & 3);
1388 			*value = rtw_read8(rtwdev, read_addr);
1389 			return 0;
1390 		}
1391 
1392 		udelay(10);
1393 	}
1394 
1395 	WARN(1, "failed to read DBI register, addr=0x%04x\n", addr);
1396 	return -EIO;
1397 }
1398 
1399 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1)
1400 {
1401 	u8 page;
1402 	u8 wflag;
1403 	u8 cnt;
1404 
1405 	rtw_write16(rtwdev, REG_MDIO_V1, data);
1406 
1407 	page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1;
1408 	page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2;
1409 	rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK);
1410 	rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page);
1411 	rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1);
1412 
1413 	for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1414 		wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG,
1415 					BIT_MDIO_WFLAG_V1);
1416 		if (wflag == 0)
1417 			return;
1418 
1419 		udelay(10);
1420 	}
1421 
1422 	WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr);
1423 }
1424 
1425 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable)
1426 {
1427 	u8 value;
1428 	int ret;
1429 
1430 	if (rtw_pci_disable_aspm)
1431 		return;
1432 
1433 	ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1434 	if (ret) {
1435 		rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1436 		return;
1437 	}
1438 
1439 	if (enable)
1440 		value |= BIT_CLKREQ_SW_EN;
1441 	else
1442 		value &= ~BIT_CLKREQ_SW_EN;
1443 
1444 	rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1445 }
1446 
1447 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable)
1448 {
1449 	u8 value;
1450 	int ret;
1451 
1452 	ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1453 	if (ret) {
1454 		rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1455 		return;
1456 	}
1457 
1458 	if (enable)
1459 		value &= ~BIT_CLKREQ_N_PAD;
1460 	else
1461 		value |= BIT_CLKREQ_N_PAD;
1462 
1463 	rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1464 }
1465 
1466 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable)
1467 {
1468 	u8 value;
1469 	int ret;
1470 
1471 	if (rtw_pci_disable_aspm)
1472 		return;
1473 
1474 	ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1475 	if (ret) {
1476 		rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret);
1477 		return;
1478 	}
1479 
1480 	if (enable)
1481 		value |= BIT_L1_SW_EN;
1482 	else
1483 		value &= ~BIT_L1_SW_EN;
1484 
1485 	rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1486 }
1487 
1488 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)
1489 {
1490 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1491 
1492 	/* Like CLKREQ, ASPM is also implemented by two HW modules, and can
1493 	 * only be enabled when host supports it.
1494 	 *
1495 	 * And ASPM mechanism should be enabled when driver/firmware enters
1496 	 * power save mode, without having heavy traffic. Because we've
1497 	 * experienced some inter-operability issues that the link tends
1498 	 * to enter L1 state on the fly even when driver is having high
1499 	 * throughput. This is probably because the ASPM behavior slightly
1500 	 * varies from different SOC.
1501 	 */
1502 	if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))
1503 		return;
1504 
1505 	if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||
1506 	    (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))
1507 		rtw_pci_aspm_set(rtwdev, enter);
1508 }
1509 
1510 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)
1511 {
1512 	struct rtw_chip_info *chip = rtwdev->chip;
1513 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1514 	struct pci_dev *pdev = rtwpci->pdev;
1515 	u16 link_ctrl;
1516 	int ret;
1517 
1518 	/* RTL8822CE has enabled REFCLK auto calibration, it does not need
1519 	 * to add clock delay to cover the REFCLK timing gap.
1520 	 */
1521 	if (chip->id == RTW_CHIP_TYPE_8822C)
1522 		rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0);
1523 
1524 	/* Though there is standard PCIE configuration space to set the
1525 	 * link control register, but by Realtek's design, driver should
1526 	 * check if host supports CLKREQ/ASPM to enable the HW module.
1527 	 *
1528 	 * These functions are implemented by two HW modules associated,
1529 	 * one is responsible to access PCIE configuration space to
1530 	 * follow the host settings, and another is in charge of doing
1531 	 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
1532 	 * the host does not support it, and due to some reasons or wrong
1533 	 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
1534 	 * loss if HW misbehaves on the link.
1535 	 *
1536 	 * Hence it's designed that driver should first check the PCIE
1537 	 * configuration space is sync'ed and enabled, then driver can turn
1538 	 * on the other module that is actually working on the mechanism.
1539 	 */
1540 	ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
1541 	if (ret) {
1542 		rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
1543 		return;
1544 	}
1545 
1546 	if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
1547 		rtw_pci_clkreq_set(rtwdev, true);
1548 
1549 	rtwpci->link_ctrl = link_ctrl;
1550 }
1551 
1552 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)
1553 {
1554 	struct rtw_chip_info *chip = rtwdev->chip;
1555 
1556 	switch (chip->id) {
1557 	case RTW_CHIP_TYPE_8822C:
1558 		if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D)
1559 			rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG,
1560 					 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1);
1561 		break;
1562 	default:
1563 		break;
1564 	}
1565 }
1566 
1567 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev)
1568 {
1569 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1570 	struct rtw_chip_info *chip = rtwdev->chip;
1571 	struct pci_dev *pdev = rtwpci->pdev;
1572 	const struct rtw_intf_phy_para *para;
1573 	u16 cut;
1574 	u16 value;
1575 	u16 offset;
1576 	int i;
1577 	int ret;
1578 
1579 	cut = BIT(0) << rtwdev->hal.cut_version;
1580 
1581 	for (i = 0; i < chip->intf_table->n_gen1_para; i++) {
1582 		para = &chip->intf_table->gen1_para[i];
1583 		if (!(para->cut_mask & cut))
1584 			continue;
1585 		if (para->offset == 0xffff)
1586 			break;
1587 		offset = para->offset;
1588 		value = para->value;
1589 		if (para->ip_sel == RTW_IP_SEL_PHY)
1590 			rtw_mdio_write(rtwdev, offset, value, true);
1591 		else
1592 			rtw_dbi_write8(rtwdev, offset, value);
1593 	}
1594 
1595 	for (i = 0; i < chip->intf_table->n_gen2_para; i++) {
1596 		para = &chip->intf_table->gen2_para[i];
1597 		if (!(para->cut_mask & cut))
1598 			continue;
1599 		if (para->offset == 0xffff)
1600 			break;
1601 		offset = para->offset;
1602 		value = para->value;
1603 		if (para->ip_sel == RTW_IP_SEL_PHY)
1604 			rtw_mdio_write(rtwdev, offset, value, false);
1605 		else
1606 			rtw_dbi_write8(rtwdev, offset, value);
1607 	}
1608 
1609 	rtw_pci_link_cfg(rtwdev);
1610 
1611 	/* Disable 8821ce completion timeout by default */
1612 	if (chip->id == RTW_CHIP_TYPE_8821C) {
1613 		ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
1614 					       PCI_EXP_DEVCTL2_COMP_TMOUT_DIS);
1615 		if (ret)
1616 			rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n",
1617 				ret);
1618 	}
1619 }
1620 
1621 static int __maybe_unused rtw_pci_suspend(struct device *dev)
1622 {
1623 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
1624 	struct rtw_dev *rtwdev = hw->priv;
1625 	struct rtw_chip_info *chip = rtwdev->chip;
1626 	struct rtw_efuse *efuse = &rtwdev->efuse;
1627 
1628 	if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1629 		rtw_pci_clkreq_pad_low(rtwdev, true);
1630 	return 0;
1631 }
1632 
1633 static int __maybe_unused rtw_pci_resume(struct device *dev)
1634 {
1635 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
1636 	struct rtw_dev *rtwdev = hw->priv;
1637 	struct rtw_chip_info *chip = rtwdev->chip;
1638 	struct rtw_efuse *efuse = &rtwdev->efuse;
1639 
1640 	if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1641 		rtw_pci_clkreq_pad_low(rtwdev, false);
1642 	return 0;
1643 }
1644 
1645 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume);
1646 EXPORT_SYMBOL(rtw_pm_ops);
1647 
1648 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1649 {
1650 	int ret;
1651 
1652 	ret = pci_enable_device(pdev);
1653 	if (ret) {
1654 		rtw_err(rtwdev, "failed to enable pci device\n");
1655 		return ret;
1656 	}
1657 
1658 	pci_set_master(pdev);
1659 	pci_set_drvdata(pdev, rtwdev->hw);
1660 	SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
1661 
1662 	return 0;
1663 }
1664 
1665 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1666 {
1667 	pci_clear_master(pdev);
1668 	pci_disable_device(pdev);
1669 }
1670 
1671 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1672 {
1673 	struct rtw_pci *rtwpci;
1674 	int ret;
1675 
1676 	rtwpci = (struct rtw_pci *)rtwdev->priv;
1677 	rtwpci->pdev = pdev;
1678 
1679 	/* after this driver can access to hw registers */
1680 	ret = rtw_pci_io_mapping(rtwdev, pdev);
1681 	if (ret) {
1682 		rtw_err(rtwdev, "failed to request pci io region\n");
1683 		goto err_out;
1684 	}
1685 
1686 	ret = rtw_pci_init(rtwdev);
1687 	if (ret) {
1688 		rtw_err(rtwdev, "failed to allocate pci resources\n");
1689 		goto err_io_unmap;
1690 	}
1691 
1692 	return 0;
1693 
1694 err_io_unmap:
1695 	rtw_pci_io_unmapping(rtwdev, pdev);
1696 
1697 err_out:
1698 	return ret;
1699 }
1700 
1701 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1702 {
1703 	rtw_pci_deinit(rtwdev);
1704 	rtw_pci_io_unmapping(rtwdev, pdev);
1705 }
1706 
1707 static struct rtw_hci_ops rtw_pci_ops = {
1708 	.tx_write = rtw_pci_tx_write,
1709 	.tx_kick_off = rtw_pci_tx_kick_off,
1710 	.flush_queues = rtw_pci_flush_queues,
1711 	.setup = rtw_pci_setup,
1712 	.start = rtw_pci_start,
1713 	.stop = rtw_pci_stop,
1714 	.deep_ps = rtw_pci_deep_ps,
1715 	.link_ps = rtw_pci_link_ps,
1716 	.interface_cfg = rtw_pci_interface_cfg,
1717 
1718 	.read8 = rtw_pci_read8,
1719 	.read16 = rtw_pci_read16,
1720 	.read32 = rtw_pci_read32,
1721 	.write8 = rtw_pci_write8,
1722 	.write16 = rtw_pci_write16,
1723 	.write32 = rtw_pci_write32,
1724 	.write_data_rsvd_page = rtw_pci_write_data_rsvd_page,
1725 	.write_data_h2c = rtw_pci_write_data_h2c,
1726 };
1727 
1728 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1729 {
1730 	unsigned int flags = PCI_IRQ_LEGACY;
1731 	int ret;
1732 
1733 	if (!rtw_disable_msi)
1734 		flags |= PCI_IRQ_MSI;
1735 
1736 	ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
1737 	if (ret < 0) {
1738 		rtw_err(rtwdev, "failed to alloc PCI irq vectors\n");
1739 		return ret;
1740 	}
1741 
1742 	ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
1743 					rtw_pci_interrupt_handler,
1744 					rtw_pci_interrupt_threadfn,
1745 					IRQF_SHARED, KBUILD_MODNAME, rtwdev);
1746 	if (ret) {
1747 		rtw_err(rtwdev, "failed to request irq %d\n", ret);
1748 		pci_free_irq_vectors(pdev);
1749 	}
1750 
1751 	return ret;
1752 }
1753 
1754 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1755 {
1756 	devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
1757 	pci_free_irq_vectors(pdev);
1758 }
1759 
1760 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget)
1761 {
1762 	struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);
1763 	struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,
1764 					      priv);
1765 	int work_done = 0;
1766 
1767 	if (rtwpci->rx_no_aspm)
1768 		rtw_pci_link_ps(rtwdev, false);
1769 
1770 	while (work_done < budget) {
1771 		u32 work_done_once;
1772 
1773 		work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,
1774 						 budget - work_done);
1775 		if (work_done_once == 0)
1776 			break;
1777 		work_done += work_done_once;
1778 	}
1779 	if (work_done < budget) {
1780 		napi_complete_done(napi, work_done);
1781 		spin_lock_bh(&rtwpci->irq_lock);
1782 		if (rtwpci->running)
1783 			rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
1784 		spin_unlock_bh(&rtwpci->irq_lock);
1785 		/* When ISR happens during polling and before napi_complete
1786 		 * while no further data is received. Data on the dma_ring will
1787 		 * not be processed immediately. Check whether dma ring is
1788 		 * empty and perform napi_schedule accordingly.
1789 		 */
1790 		if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))
1791 			napi_schedule(napi);
1792 	}
1793 	if (rtwpci->rx_no_aspm)
1794 		rtw_pci_link_ps(rtwdev, true);
1795 
1796 	return work_done;
1797 }
1798 
1799 static void rtw_pci_napi_init(struct rtw_dev *rtwdev)
1800 {
1801 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1802 
1803 	init_dummy_netdev(&rtwpci->netdev);
1804 	netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);
1805 }
1806 
1807 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
1808 {
1809 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1810 
1811 	rtw_pci_napi_stop(rtwdev);
1812 	netif_napi_del(&rtwpci->napi);
1813 }
1814 
1815 int rtw_pci_probe(struct pci_dev *pdev,
1816 		  const struct pci_device_id *id)
1817 {
1818 	struct pci_dev *bridge = pci_upstream_bridge(pdev);
1819 	struct ieee80211_hw *hw;
1820 	struct rtw_dev *rtwdev;
1821 	struct rtw_pci *rtwpci;
1822 	int drv_data_size;
1823 	int ret;
1824 
1825 	drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci);
1826 	hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops);
1827 	if (!hw) {
1828 		dev_err(&pdev->dev, "failed to allocate hw\n");
1829 		return -ENOMEM;
1830 	}
1831 
1832 	rtwdev = hw->priv;
1833 	rtwdev->hw = hw;
1834 	rtwdev->dev = &pdev->dev;
1835 	rtwdev->chip = (struct rtw_chip_info *)id->driver_data;
1836 	rtwdev->hci.ops = &rtw_pci_ops;
1837 	rtwdev->hci.type = RTW_HCI_TYPE_PCIE;
1838 
1839 	rtwpci = (struct rtw_pci *)rtwdev->priv;
1840 	atomic_set(&rtwpci->link_usage, 1);
1841 
1842 	ret = rtw_core_init(rtwdev);
1843 	if (ret)
1844 		goto err_release_hw;
1845 
1846 	rtw_dbg(rtwdev, RTW_DBG_PCI,
1847 		"rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n",
1848 		pdev->vendor, pdev->device, pdev->revision);
1849 
1850 	ret = rtw_pci_claim(rtwdev, pdev);
1851 	if (ret) {
1852 		rtw_err(rtwdev, "failed to claim pci device\n");
1853 		goto err_deinit_core;
1854 	}
1855 
1856 	ret = rtw_pci_setup_resource(rtwdev, pdev);
1857 	if (ret) {
1858 		rtw_err(rtwdev, "failed to setup pci resources\n");
1859 		goto err_pci_declaim;
1860 	}
1861 
1862 	rtw_pci_napi_init(rtwdev);
1863 
1864 	ret = rtw_chip_info_setup(rtwdev);
1865 	if (ret) {
1866 		rtw_err(rtwdev, "failed to setup chip information\n");
1867 		goto err_destroy_pci;
1868 	}
1869 
1870 	/* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */
1871 	if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL)
1872 		rtwpci->rx_no_aspm = true;
1873 
1874 	rtw_pci_phy_cfg(rtwdev);
1875 
1876 	ret = rtw_register_hw(rtwdev, hw);
1877 	if (ret) {
1878 		rtw_err(rtwdev, "failed to register hw\n");
1879 		goto err_destroy_pci;
1880 	}
1881 
1882 	ret = rtw_pci_request_irq(rtwdev, pdev);
1883 	if (ret) {
1884 		ieee80211_unregister_hw(hw);
1885 		goto err_destroy_pci;
1886 	}
1887 
1888 	return 0;
1889 
1890 err_destroy_pci:
1891 	rtw_pci_napi_deinit(rtwdev);
1892 	rtw_pci_destroy(rtwdev, pdev);
1893 
1894 err_pci_declaim:
1895 	rtw_pci_declaim(rtwdev, pdev);
1896 
1897 err_deinit_core:
1898 	rtw_core_deinit(rtwdev);
1899 
1900 err_release_hw:
1901 	ieee80211_free_hw(hw);
1902 
1903 	return ret;
1904 }
1905 EXPORT_SYMBOL(rtw_pci_probe);
1906 
1907 void rtw_pci_remove(struct pci_dev *pdev)
1908 {
1909 	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1910 	struct rtw_dev *rtwdev;
1911 	struct rtw_pci *rtwpci;
1912 
1913 	if (!hw)
1914 		return;
1915 
1916 	rtwdev = hw->priv;
1917 	rtwpci = (struct rtw_pci *)rtwdev->priv;
1918 
1919 	rtw_unregister_hw(rtwdev, hw);
1920 	rtw_pci_disable_interrupt(rtwdev, rtwpci);
1921 	rtw_pci_napi_deinit(rtwdev);
1922 	rtw_pci_destroy(rtwdev, pdev);
1923 	rtw_pci_declaim(rtwdev, pdev);
1924 	rtw_pci_free_irq(rtwdev, pdev);
1925 	rtw_core_deinit(rtwdev);
1926 	ieee80211_free_hw(hw);
1927 }
1928 EXPORT_SYMBOL(rtw_pci_remove);
1929 
1930 void rtw_pci_shutdown(struct pci_dev *pdev)
1931 {
1932 	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1933 	struct rtw_dev *rtwdev;
1934 	struct rtw_chip_info *chip;
1935 
1936 	if (!hw)
1937 		return;
1938 
1939 	rtwdev = hw->priv;
1940 	chip = rtwdev->chip;
1941 
1942 	if (chip->ops->shutdown)
1943 		chip->ops->shutdown(rtwdev);
1944 
1945 	pci_set_power_state(pdev, PCI_D3hot);
1946 }
1947 EXPORT_SYMBOL(rtw_pci_shutdown);
1948 
1949 MODULE_AUTHOR("Realtek Corporation");
1950 MODULE_DESCRIPTION("Realtek 802.11ac wireless PCI driver");
1951 MODULE_LICENSE("Dual BSD/GPL");
1952 #if defined(__FreeBSD__)
1953 MODULE_VERSION(rtw_pci, 1);
1954 MODULE_DEPEND(rtw_pci, linuxkpi, 1, 1, 1);
1955 MODULE_DEPEND(rtw_pci, linuxkpi_wlan, 1, 1, 1);
1956 #ifdef CONFIG_RTW88_DEBUGFS
1957 MODULE_DEPEND(rtw_pci, lindebugfs, 1, 1, 1);
1958 #endif
1959 #endif
1960