xref: /linux/drivers/net/wireless/realtek/rtw89/pci.c (revision 59fff63cc2b75dcfe08f9eeb4b2187d73e53843d)
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2020  Realtek Corporation
3  */
4 
5 #include <linux/pci.h>
6 
7 #include "mac.h"
8 #include "pci.h"
9 #include "reg.h"
10 #include "ser.h"
11 
12 static bool rtw89_pci_disable_clkreq;
13 static bool rtw89_pci_disable_aspm_l1;
14 static bool rtw89_pci_disable_l1ss;
15 module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
16 module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
17 module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
18 MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
19 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
20 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
21 
22 static int rtw89_pci_rst_bdram_pcie(struct rtw89_dev *rtwdev)
23 {
24 	u32 val;
25 	int ret;
26 
27 	rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1,
28 		      rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) | B_AX_RST_BDRAM);
29 
30 	ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
31 				       1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
32 				       rtwdev, R_AX_PCIE_INIT_CFG1);
33 
34 	if (ret)
35 		return -EBUSY;
36 
37 	return 0;
38 }
39 
40 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
41 				struct rtw89_pci_dma_ring *bd_ring,
42 				u32 cur_idx, bool tx)
43 {
44 	u32 cnt, cur_rp, wp, rp, len;
45 
46 	rp = bd_ring->rp;
47 	wp = bd_ring->wp;
48 	len = bd_ring->len;
49 
50 	cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
51 	if (tx)
52 		cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
53 	else
54 		cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
55 
56 	bd_ring->rp = cur_rp;
57 
58 	return cnt;
59 }
60 
61 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
62 				 struct rtw89_pci_tx_ring *tx_ring)
63 {
64 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
65 	u32 addr_idx = bd_ring->addr.idx;
66 	u32 cnt, idx;
67 
68 	idx = rtw89_read32(rtwdev, addr_idx);
69 	cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
70 
71 	return cnt;
72 }
73 
74 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
75 				    struct rtw89_pci *rtwpci,
76 				    u32 cnt, bool release_all)
77 {
78 	struct rtw89_pci_tx_data *tx_data;
79 	struct sk_buff *skb;
80 	u32 qlen;
81 
82 	while (cnt--) {
83 		skb = skb_dequeue(&rtwpci->h2c_queue);
84 		if (!skb) {
85 			rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
86 			return;
87 		}
88 		skb_queue_tail(&rtwpci->h2c_release_queue, skb);
89 	}
90 
91 	qlen = skb_queue_len(&rtwpci->h2c_release_queue);
92 	if (!release_all)
93 	       qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
94 
95 	while (qlen--) {
96 		skb = skb_dequeue(&rtwpci->h2c_release_queue);
97 		if (!skb) {
98 			rtw89_err(rtwdev, "failed to release fwcmd\n");
99 			return;
100 		}
101 		tx_data = RTW89_PCI_TX_SKB_CB(skb);
102 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
103 				 DMA_TO_DEVICE);
104 		dev_kfree_skb_any(skb);
105 	}
106 }
107 
108 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
109 				       struct rtw89_pci *rtwpci)
110 {
111 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
112 	u32 cnt;
113 
114 	cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
115 	if (!cnt)
116 		return;
117 	rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
118 }
119 
120 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
121 				 struct rtw89_pci_rx_ring *rx_ring)
122 {
123 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
124 	u32 addr_idx = bd_ring->addr.idx;
125 	u32 cnt, idx;
126 
127 	idx = rtw89_read32(rtwdev, addr_idx);
128 	cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
129 
130 	return cnt;
131 }
132 
133 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
134 				       struct sk_buff *skb)
135 {
136 	struct rtw89_pci_rx_info *rx_info;
137 	dma_addr_t dma;
138 
139 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
140 	dma = rx_info->dma;
141 	dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
142 				DMA_FROM_DEVICE);
143 }
144 
145 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
146 					  struct sk_buff *skb)
147 {
148 	struct rtw89_pci_rx_info *rx_info;
149 	dma_addr_t dma;
150 
151 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
152 	dma = rx_info->dma;
153 	dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
154 				   DMA_FROM_DEVICE);
155 }
156 
157 static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
158 				      struct sk_buff *skb)
159 {
160 	struct rtw89_pci_rxbd_info *rxbd_info;
161 	struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
162 
163 	rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
164 	rx_info->fs = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_FS);
165 	rx_info->ls = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_LS);
166 	rx_info->len = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_WRITE_SIZE);
167 	rx_info->tag = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_TAG);
168 
169 	return 0;
170 }
171 
172 static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
173 {
174 	const struct rtw89_pci_info *info = rtwdev->pci_info;
175 	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
176 	const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
177 
178 	if (enable) {
179 		rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
180 		if (dma_stop2->addr)
181 			rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
182 	} else {
183 		rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
184 		if (dma_stop2->addr)
185 			rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
186 	}
187 }
188 
189 static void rtw89_pci_ctrl_txdma_fw_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
190 {
191 	const struct rtw89_pci_info *info = rtwdev->pci_info;
192 	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
193 
194 	if (enable)
195 		rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
196 	else
197 		rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
198 }
199 
200 static bool
201 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
202 		      struct sk_buff *new,
203 		      const struct sk_buff *skb, u32 offset,
204 		      const struct rtw89_pci_rx_info *rx_info,
205 		      const struct rtw89_rx_desc_info *desc_info)
206 {
207 	u32 copy_len = rx_info->len - offset;
208 
209 	if (unlikely(skb_tailroom(new) < copy_len)) {
210 		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
211 			    "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
212 			    rx_info->len, desc_info->pkt_size, offset, fs, ls);
213 		rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
214 			       skb->data, rx_info->len);
215 		/* length of a single segment skb is desc_info->pkt_size */
216 		if (fs && ls) {
217 			copy_len = desc_info->pkt_size;
218 		} else {
219 			rtw89_info(rtwdev, "drop rx data due to invalid length\n");
220 			return false;
221 		}
222 	}
223 
224 	skb_put_data(new, skb->data + offset, copy_len);
225 
226 	return true;
227 }
228 
229 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
230 				       struct rtw89_pci_rx_ring *rx_ring)
231 {
232 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
233 	struct rtw89_pci_rx_info *rx_info;
234 	struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
235 	struct sk_buff *new = rx_ring->diliver_skb;
236 	struct sk_buff *skb;
237 	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
238 	u32 offset;
239 	u32 cnt = 1;
240 	bool fs, ls;
241 	int ret;
242 
243 	skb = rx_ring->buf[bd_ring->wp];
244 	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
245 
246 	ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
247 	if (ret) {
248 		rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
249 			  bd_ring->wp, ret);
250 		goto err_sync_device;
251 	}
252 
253 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
254 	fs = rx_info->fs;
255 	ls = rx_info->ls;
256 
257 	if (fs) {
258 		if (new) {
259 			rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
260 				    "skb should not be ready before first segment start\n");
261 			goto err_sync_device;
262 		}
263 		if (desc_info->ready) {
264 			rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
265 			goto err_sync_device;
266 		}
267 
268 		rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
269 
270 		new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
271 		if (!new)
272 			goto err_sync_device;
273 
274 		rx_ring->diliver_skb = new;
275 
276 		/* first segment has RX desc */
277 		offset = desc_info->offset + desc_info->rxd_len;
278 	} else {
279 		offset = sizeof(struct rtw89_pci_rxbd_info);
280 		if (!new) {
281 			rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
282 			goto err_sync_device;
283 		}
284 	}
285 	if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
286 		goto err_sync_device;
287 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
288 	rtw89_pci_rxbd_increase(rx_ring, 1);
289 
290 	if (!desc_info->ready) {
291 		rtw89_warn(rtwdev, "no rx desc information\n");
292 		goto err_free_resource;
293 	}
294 	if (ls) {
295 		rtw89_core_rx(rtwdev, desc_info, new);
296 		rx_ring->diliver_skb = NULL;
297 		desc_info->ready = false;
298 	}
299 
300 	return cnt;
301 
302 err_sync_device:
303 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
304 	rtw89_pci_rxbd_increase(rx_ring, 1);
305 err_free_resource:
306 	if (new)
307 		dev_kfree_skb_any(new);
308 	rx_ring->diliver_skb = NULL;
309 	desc_info->ready = false;
310 
311 	return cnt;
312 }
313 
314 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
315 				   struct rtw89_pci_rx_ring *rx_ring,
316 				   u32 cnt)
317 {
318 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
319 	u32 rx_cnt;
320 
321 	while (cnt && rtwdev->napi_budget_countdown > 0) {
322 		rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
323 		if (!rx_cnt) {
324 			rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
325 
326 			/* skip the rest RXBD bufs */
327 			rtw89_pci_rxbd_increase(rx_ring, cnt);
328 			break;
329 		}
330 
331 		cnt -= rx_cnt;
332 	}
333 
334 	rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
335 }
336 
337 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
338 				  struct rtw89_pci *rtwpci, int budget)
339 {
340 	struct rtw89_pci_rx_ring *rx_ring;
341 	int countdown = rtwdev->napi_budget_countdown;
342 	u32 cnt;
343 
344 	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
345 
346 	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
347 	if (!cnt)
348 		return 0;
349 
350 	cnt = min_t(u32, budget, cnt);
351 
352 	rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
353 
354 	/* In case of flushing pending SKBs, the countdown may exceed. */
355 	if (rtwdev->napi_budget_countdown <= 0)
356 		return budget;
357 
358 	return budget - countdown;
359 }
360 
361 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
362 				struct rtw89_pci_tx_ring *tx_ring,
363 				struct sk_buff *skb, u8 tx_status)
364 {
365 	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
366 	struct ieee80211_tx_info *info;
367 
368 	rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
369 
370 	info = IEEE80211_SKB_CB(skb);
371 	ieee80211_tx_info_clear_status(info);
372 
373 	if (info->flags & IEEE80211_TX_CTL_NO_ACK)
374 		info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
375 	if (tx_status == RTW89_TX_DONE) {
376 		info->flags |= IEEE80211_TX_STAT_ACK;
377 		tx_ring->tx_acked++;
378 	} else {
379 		if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
380 			rtw89_debug(rtwdev, RTW89_DBG_FW,
381 				    "failed to TX of status %x\n", tx_status);
382 		switch (tx_status) {
383 		case RTW89_TX_RETRY_LIMIT:
384 			tx_ring->tx_retry_lmt++;
385 			break;
386 		case RTW89_TX_LIFE_TIME:
387 			tx_ring->tx_life_time++;
388 			break;
389 		case RTW89_TX_MACID_DROP:
390 			tx_ring->tx_mac_id_drop++;
391 			break;
392 		default:
393 			rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
394 			break;
395 		}
396 	}
397 
398 	ieee80211_tx_status_ni(rtwdev->hw, skb);
399 }
400 
401 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
402 {
403 	struct rtw89_pci_tx_wd *txwd;
404 	u32 cnt;
405 
406 	cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
407 	while (cnt--) {
408 		txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
409 		if (!txwd) {
410 			rtw89_warn(rtwdev, "No busy txwd pages available\n");
411 			break;
412 		}
413 
414 		list_del_init(&txwd->list);
415 
416 		/* this skb has been freed by RPP */
417 		if (skb_queue_len(&txwd->queue) == 0)
418 			rtw89_pci_enqueue_txwd(tx_ring, txwd);
419 	}
420 }
421 
422 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
423 					struct rtw89_pci_tx_ring *tx_ring)
424 {
425 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
426 	struct rtw89_pci_tx_wd *txwd;
427 	int i;
428 
429 	for (i = 0; i < wd_ring->page_num; i++) {
430 		txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
431 		if (!txwd)
432 			break;
433 
434 		list_del_init(&txwd->list);
435 	}
436 }
437 
438 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
439 				       struct rtw89_pci_tx_ring *tx_ring,
440 				       struct rtw89_pci_tx_wd *txwd, u16 seq,
441 				       u8 tx_status)
442 {
443 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
444 	struct rtw89_pci_tx_data *tx_data;
445 	struct sk_buff *skb, *tmp;
446 	u8 txch = tx_ring->txch;
447 
448 	if (!list_empty(&txwd->list)) {
449 		rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
450 		/* In low power mode, RPP can receive before updating of TX BD.
451 		 * In normal mode, it should not happen so give it a warning.
452 		 */
453 		if (!rtwpci->low_power && !list_empty(&txwd->list))
454 			rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
455 				   txch, seq);
456 	}
457 
458 	skb_queue_walk_safe(&txwd->queue, skb, tmp) {
459 		skb_unlink(skb, &txwd->queue);
460 
461 		tx_data = RTW89_PCI_TX_SKB_CB(skb);
462 		dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
463 				 DMA_TO_DEVICE);
464 
465 		rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
466 	}
467 
468 	if (list_empty(&txwd->list))
469 		rtw89_pci_enqueue_txwd(tx_ring, txwd);
470 }
471 
472 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
473 				  struct rtw89_pci_rpp_fmt *rpp)
474 {
475 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
476 	struct rtw89_pci_tx_ring *tx_ring;
477 	struct rtw89_pci_tx_wd_ring *wd_ring;
478 	struct rtw89_pci_tx_wd *txwd;
479 	u16 seq;
480 	u8 qsel, tx_status, txch;
481 
482 	seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ);
483 	qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL);
484 	tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS);
485 	txch = rtw89_core_get_ch_dma(rtwdev, qsel);
486 
487 	if (txch == RTW89_TXCH_CH12) {
488 		rtw89_warn(rtwdev, "should no fwcmd release report\n");
489 		return;
490 	}
491 
492 	tx_ring = &rtwpci->tx_rings[txch];
493 	wd_ring = &tx_ring->wd_ring;
494 	txwd = &wd_ring->pages[seq];
495 
496 	rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
497 }
498 
499 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
500 					       struct rtw89_pci_tx_ring *tx_ring)
501 {
502 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
503 	struct rtw89_pci_tx_wd *txwd;
504 	int i;
505 
506 	for (i = 0; i < wd_ring->page_num; i++) {
507 		txwd = &wd_ring->pages[i];
508 
509 		if (!list_empty(&txwd->list))
510 			continue;
511 
512 		rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
513 	}
514 }
515 
516 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
517 				     struct rtw89_pci_rx_ring *rx_ring,
518 				     u32 max_cnt)
519 {
520 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
521 	struct rtw89_pci_rx_info *rx_info;
522 	struct rtw89_pci_rpp_fmt *rpp;
523 	struct rtw89_rx_desc_info desc_info = {};
524 	struct sk_buff *skb;
525 	u32 cnt = 0;
526 	u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
527 	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
528 	u32 offset;
529 	int ret;
530 
531 	skb = rx_ring->buf[bd_ring->wp];
532 	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
533 
534 	ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
535 	if (ret) {
536 		rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
537 			  bd_ring->wp, ret);
538 		goto err_sync_device;
539 	}
540 
541 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
542 	if (!rx_info->fs || !rx_info->ls) {
543 		rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
544 		return cnt;
545 	}
546 
547 	rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
548 
549 	/* first segment has RX desc */
550 	offset = desc_info.offset + desc_info.rxd_len;
551 	for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
552 		rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset);
553 		rtw89_pci_release_rpp(rtwdev, rpp);
554 	}
555 
556 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
557 	rtw89_pci_rxbd_increase(rx_ring, 1);
558 	cnt++;
559 
560 	return cnt;
561 
562 err_sync_device:
563 	rtw89_pci_sync_skb_for_device(rtwdev, skb);
564 	return 0;
565 }
566 
567 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
568 				 struct rtw89_pci_rx_ring *rx_ring,
569 				 u32 cnt)
570 {
571 	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
572 	u32 release_cnt;
573 
574 	while (cnt) {
575 		release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
576 		if (!release_cnt) {
577 			rtw89_err(rtwdev, "failed to release TX skbs\n");
578 
579 			/* skip the rest RXBD bufs */
580 			rtw89_pci_rxbd_increase(rx_ring, cnt);
581 			break;
582 		}
583 
584 		cnt -= release_cnt;
585 	}
586 
587 	rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
588 }
589 
590 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
591 				  struct rtw89_pci *rtwpci, int budget)
592 {
593 	struct rtw89_pci_rx_ring *rx_ring;
594 	u32 cnt;
595 	int work_done;
596 
597 	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
598 
599 	spin_lock_bh(&rtwpci->trx_lock);
600 
601 	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
602 	if (cnt == 0)
603 		goto out_unlock;
604 
605 	rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
606 
607 out_unlock:
608 	spin_unlock_bh(&rtwpci->trx_lock);
609 
610 	/* always release all RPQ */
611 	work_done = min_t(int, cnt, budget);
612 	rtwdev->napi_budget_countdown -= work_done;
613 
614 	return work_done;
615 }
616 
617 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
618 				      struct rtw89_pci *rtwpci)
619 {
620 	struct rtw89_pci_rx_ring *rx_ring;
621 	struct rtw89_pci_dma_ring *bd_ring;
622 	u32 reg_idx;
623 	u16 hw_idx, hw_idx_next, host_idx;
624 	int i;
625 
626 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
627 		rx_ring = &rtwpci->rx_rings[i];
628 		bd_ring = &rx_ring->bd_ring;
629 
630 		reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
631 		hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
632 		host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
633 		hw_idx_next = (hw_idx + 1) % bd_ring->len;
634 
635 		if (hw_idx_next == host_idx)
636 			rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
637 
638 		rtw89_debug(rtwdev, RTW89_DBG_TXRX,
639 			    "%d RXD unavailable, idx=0x%08x, len=%d\n",
640 			    i, reg_idx, bd_ring->len);
641 	}
642 }
643 
644 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
645 			       struct rtw89_pci *rtwpci,
646 			       struct rtw89_pci_isrs *isrs)
647 {
648 	isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
649 	isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
650 	isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
651 
652 	rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
653 	rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
654 	rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
655 }
656 EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
657 
658 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
659 				  struct rtw89_pci *rtwpci,
660 				  struct rtw89_pci_isrs *isrs)
661 {
662 	isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
663 	isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
664 			      rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
665 	isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
666 			rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
667 	isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
668 			rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
669 
670 	if (isrs->halt_c2h_isrs)
671 		rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
672 	if (isrs->isrs[0])
673 		rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
674 	if (isrs->isrs[1])
675 		rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
676 }
677 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
678 
679 static void rtw89_pci_clear_isr0(struct rtw89_dev *rtwdev, u32 isr00)
680 {
681 	/* write 1 clear */
682 	rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isr00);
683 }
684 
685 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
686 {
687 	rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
688 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
689 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
690 }
691 EXPORT_SYMBOL(rtw89_pci_enable_intr);
692 
693 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
694 {
695 	rtw89_write32(rtwdev, R_AX_HIMR0, 0);
696 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
697 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
698 }
699 EXPORT_SYMBOL(rtw89_pci_disable_intr);
700 
701 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
702 {
703 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
704 	rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
705 	rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
706 	rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
707 }
708 EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
709 
710 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
711 {
712 	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
713 }
714 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
715 
716 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
717 {
718 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
719 	unsigned long flags;
720 
721 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
722 	rtw89_chip_disable_intr(rtwdev, rtwpci);
723 	rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
724 	rtw89_chip_enable_intr(rtwdev, rtwpci);
725 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
726 }
727 
728 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
729 {
730 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
731 	unsigned long flags;
732 
733 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
734 	rtw89_chip_disable_intr(rtwdev, rtwpci);
735 	rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
736 	rtw89_chip_enable_intr(rtwdev, rtwpci);
737 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
738 }
739 
740 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
741 {
742 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
743 	int budget = NAPI_POLL_WEIGHT;
744 
745 	/* To prevent RXQ get stuck due to run out of budget. */
746 	rtwdev->napi_budget_countdown = budget;
747 
748 	rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
749 	rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
750 }
751 
752 static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
753 {
754 	struct rtw89_dev *rtwdev = dev;
755 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
756 	struct rtw89_pci_isrs isrs;
757 	unsigned long flags;
758 
759 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
760 	rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
761 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
762 
763 	if (unlikely(isrs.isrs[0] & B_AX_RDU_INT))
764 		rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
765 
766 	if (unlikely(isrs.halt_c2h_isrs & B_AX_HALT_C2H_INT_EN))
767 		rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
768 
769 	if (unlikely(isrs.halt_c2h_isrs & B_AX_WDT_TIMEOUT_INT_EN))
770 		rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
771 
772 	if (unlikely(rtwpci->under_recovery))
773 		goto enable_intr;
774 
775 	if (unlikely(rtwpci->low_power)) {
776 		rtw89_pci_low_power_interrupt_handler(rtwdev);
777 		goto enable_intr;
778 	}
779 
780 	if (likely(rtwpci->running)) {
781 		local_bh_disable();
782 		napi_schedule(&rtwdev->napi);
783 		local_bh_enable();
784 	}
785 
786 	return IRQ_HANDLED;
787 
788 enable_intr:
789 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
790 	if (likely(rtwpci->running))
791 		rtw89_chip_enable_intr(rtwdev, rtwpci);
792 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
793 	return IRQ_HANDLED;
794 }
795 
796 static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
797 {
798 	struct rtw89_dev *rtwdev = dev;
799 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
800 	unsigned long flags;
801 	irqreturn_t irqret = IRQ_WAKE_THREAD;
802 
803 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
804 
805 	/* If interrupt event is on the road, it is still trigger interrupt
806 	 * even we have done pci_stop() to turn off IMR.
807 	 */
808 	if (unlikely(!rtwpci->running)) {
809 		irqret = IRQ_HANDLED;
810 		goto exit;
811 	}
812 
813 	rtw89_chip_disable_intr(rtwdev, rtwpci);
814 exit:
815 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
816 
817 	return irqret;
818 }
819 
820 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
821 	[RTW89_TXCH_##txch] = { \
822 		.num = R_AX_##txch##_TXBD_NUM ##v, \
823 		.idx = R_AX_##txch##_TXBD_IDX ##v, \
824 		.bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
825 		.desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
826 		.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
827 	}
828 
829 #define DEF_TXCHADDRS(info, txch, v...) \
830 	[RTW89_TXCH_##txch] = { \
831 		.num = R_AX_##txch##_TXBD_NUM, \
832 		.idx = R_AX_##txch##_TXBD_IDX, \
833 		.bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
834 		.desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
835 		.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
836 	}
837 
838 #define DEF_RXCHADDRS(info, rxch, v...) \
839 	[RTW89_RXCH_##rxch] = { \
840 		.num = R_AX_##rxch##_RXBD_NUM ##v, \
841 		.idx = R_AX_##rxch##_RXBD_IDX ##v, \
842 		.desa_l = R_AX_##rxch##_RXBD_DESA_L ##v, \
843 		.desa_h = R_AX_##rxch##_RXBD_DESA_H ##v, \
844 	}
845 
846 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
847 	.tx = {
848 		DEF_TXCHADDRS(info, ACH0),
849 		DEF_TXCHADDRS(info, ACH1),
850 		DEF_TXCHADDRS(info, ACH2),
851 		DEF_TXCHADDRS(info, ACH3),
852 		DEF_TXCHADDRS(info, ACH4),
853 		DEF_TXCHADDRS(info, ACH5),
854 		DEF_TXCHADDRS(info, ACH6),
855 		DEF_TXCHADDRS(info, ACH7),
856 		DEF_TXCHADDRS(info, CH8),
857 		DEF_TXCHADDRS(info, CH9),
858 		DEF_TXCHADDRS_TYPE1(info, CH10),
859 		DEF_TXCHADDRS_TYPE1(info, CH11),
860 		DEF_TXCHADDRS(info, CH12),
861 	},
862 	.rx = {
863 		DEF_RXCHADDRS(info, RXQ),
864 		DEF_RXCHADDRS(info, RPQ),
865 	},
866 };
867 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
868 
869 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
870 	.tx = {
871 		DEF_TXCHADDRS(info, ACH0, _V1),
872 		DEF_TXCHADDRS(info, ACH1, _V1),
873 		DEF_TXCHADDRS(info, ACH2, _V1),
874 		DEF_TXCHADDRS(info, ACH3, _V1),
875 		DEF_TXCHADDRS(info, ACH4, _V1),
876 		DEF_TXCHADDRS(info, ACH5, _V1),
877 		DEF_TXCHADDRS(info, ACH6, _V1),
878 		DEF_TXCHADDRS(info, ACH7, _V1),
879 		DEF_TXCHADDRS(info, CH8, _V1),
880 		DEF_TXCHADDRS(info, CH9, _V1),
881 		DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
882 		DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
883 		DEF_TXCHADDRS(info, CH12, _V1),
884 	},
885 	.rx = {
886 		DEF_RXCHADDRS(info, RXQ, _V1),
887 		DEF_RXCHADDRS(info, RPQ, _V1),
888 	},
889 };
890 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
891 
892 #undef DEF_TXCHADDRS_TYPE1
893 #undef DEF_TXCHADDRS
894 #undef DEF_RXCHADDRS
895 
896 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
897 				    enum rtw89_tx_channel txch,
898 				    const struct rtw89_pci_ch_dma_addr **addr)
899 {
900 	const struct rtw89_pci_info *info = rtwdev->pci_info;
901 
902 	if (txch >= RTW89_TXCH_NUM)
903 		return -EINVAL;
904 
905 	*addr = &info->dma_addr_set->tx[txch];
906 
907 	return 0;
908 }
909 
910 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
911 				    enum rtw89_rx_channel rxch,
912 				    const struct rtw89_pci_ch_dma_addr **addr)
913 {
914 	const struct rtw89_pci_info *info = rtwdev->pci_info;
915 
916 	if (rxch >= RTW89_RXCH_NUM)
917 		return -EINVAL;
918 
919 	*addr = &info->dma_addr_set->rx[rxch];
920 
921 	return 0;
922 }
923 
924 static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
925 {
926 	struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
927 
928 	/* reserved 1 desc check ring is full or not */
929 	if (bd_ring->rp > bd_ring->wp)
930 		return bd_ring->rp - bd_ring->wp - 1;
931 
932 	return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
933 }
934 
935 static
936 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
937 {
938 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
939 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
940 	u32 cnt;
941 
942 	spin_lock_bh(&rtwpci->trx_lock);
943 	rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
944 	cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
945 	spin_unlock_bh(&rtwpci->trx_lock);
946 
947 	return cnt;
948 }
949 
950 static
951 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
952 						   u8 txch)
953 {
954 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
955 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
956 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
957 	u32 cnt;
958 
959 	spin_lock_bh(&rtwpci->trx_lock);
960 	cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
961 	cnt = min(cnt, wd_ring->curr_num);
962 	spin_unlock_bh(&rtwpci->trx_lock);
963 
964 	return cnt;
965 }
966 
967 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
968 						     u8 txch)
969 {
970 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
971 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
972 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
973 	const struct rtw89_chip_info *chip = rtwdev->chip;
974 	u32 bd_cnt, wd_cnt, min_cnt = 0;
975 	struct rtw89_pci_rx_ring *rx_ring;
976 	enum rtw89_debug_mask debug_mask;
977 	u32 cnt;
978 
979 	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
980 
981 	spin_lock_bh(&rtwpci->trx_lock);
982 	bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
983 	wd_cnt = wd_ring->curr_num;
984 
985 	if (wd_cnt == 0 || bd_cnt == 0) {
986 		cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
987 		if (cnt)
988 			rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
989 		else if (wd_cnt == 0)
990 			goto out_unlock;
991 
992 		bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
993 		if (bd_cnt == 0)
994 			rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
995 	}
996 
997 	bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
998 	wd_cnt = wd_ring->curr_num;
999 	min_cnt = min(bd_cnt, wd_cnt);
1000 	if (min_cnt == 0) {
1001 		/* This message can be frequently shown in low power mode or
1002 		 * high traffic with small FIFO chips, and we have recognized it as normal
1003 		 * behavior, so print with mask RTW89_DBG_TXRX in these situations.
1004 		 */
1005 		if (rtwpci->low_power || chip->small_fifo_size)
1006 			debug_mask = RTW89_DBG_TXRX;
1007 		else
1008 			debug_mask = RTW89_DBG_UNEXP;
1009 
1010 		rtw89_debug(rtwdev, debug_mask,
1011 			    "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
1012 			    wd_cnt, bd_cnt);
1013 	}
1014 
1015 out_unlock:
1016 	spin_unlock_bh(&rtwpci->trx_lock);
1017 
1018 	return min_cnt;
1019 }
1020 
1021 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1022 						   u8 txch)
1023 {
1024 	if (rtwdev->hci.paused)
1025 		return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1026 
1027 	if (txch == RTW89_TXCH_CH12)
1028 		return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1029 
1030 	return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1031 }
1032 
1033 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1034 {
1035 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1036 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1037 	u32 host_idx, addr;
1038 
1039 	spin_lock_bh(&rtwpci->trx_lock);
1040 
1041 	addr = bd_ring->addr.idx;
1042 	host_idx = bd_ring->wp;
1043 	rtw89_write16(rtwdev, addr, host_idx);
1044 
1045 	spin_unlock_bh(&rtwpci->trx_lock);
1046 }
1047 
1048 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1049 					int n_txbd)
1050 {
1051 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1052 	u32 host_idx, len;
1053 
1054 	len = bd_ring->len;
1055 	host_idx = bd_ring->wp + n_txbd;
1056 	host_idx = host_idx < len ? host_idx : host_idx - len;
1057 
1058 	bd_ring->wp = host_idx;
1059 }
1060 
1061 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1062 {
1063 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1064 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1065 
1066 	if (rtwdev->hci.paused) {
1067 		set_bit(txch, rtwpci->kick_map);
1068 		return;
1069 	}
1070 
1071 	__rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1072 }
1073 
1074 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1075 {
1076 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1077 	struct rtw89_pci_tx_ring *tx_ring;
1078 	int txch;
1079 
1080 	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1081 		if (!test_and_clear_bit(txch, rtwpci->kick_map))
1082 			continue;
1083 
1084 		tx_ring = &rtwpci->tx_rings[txch];
1085 		__rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1086 	}
1087 }
1088 
1089 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1090 {
1091 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1092 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1093 	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1094 	u32 cur_idx, cur_rp;
1095 	u8 i;
1096 
1097 	/* Because the time taked by the I/O is a bit dynamic, it's hard to
1098 	 * define a reasonable fixed total timeout to use read_poll_timeout*
1099 	 * helper. Instead, we can ensure a reasonable polling times, so we
1100 	 * just use for loop with udelay here.
1101 	 */
1102 	for (i = 0; i < 60; i++) {
1103 		cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1104 		cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1105 		if (cur_rp == bd_ring->wp)
1106 			return;
1107 
1108 		udelay(1);
1109 	}
1110 
1111 	if (!drop)
1112 		rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1113 }
1114 
1115 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1116 					bool drop)
1117 {
1118 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1119 	u8 i;
1120 
1121 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1122 		/* It may be unnecessary to flush FWCMD queue. */
1123 		if (i == RTW89_TXCH_CH12)
1124 			continue;
1125 		if (info->tx_dma_ch_mask & BIT(i))
1126 			continue;
1127 
1128 		if (txchs & BIT(i))
1129 			__pci_flush_txch(rtwdev, i, drop);
1130 	}
1131 }
1132 
1133 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1134 				       bool drop)
1135 {
1136 	__rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1137 }
1138 
1139 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1140 			       void *txaddr_info_addr, u32 total_len,
1141 			       dma_addr_t dma, u8 *add_info_nr)
1142 {
1143 	struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1144 
1145 	txaddr_info->length = cpu_to_le16(total_len);
1146 	txaddr_info->option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS |
1147 					  RTW89_PCI_ADDR_NUM(1));
1148 	txaddr_info->dma = cpu_to_le32(dma);
1149 
1150 	*add_info_nr = 1;
1151 
1152 	return sizeof(*txaddr_info);
1153 }
1154 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1155 
1156 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1157 				  void *txaddr_info_addr, u32 total_len,
1158 				  dma_addr_t dma, u8 *add_info_nr)
1159 {
1160 	struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1161 	u32 remain = total_len;
1162 	u32 len;
1163 	u16 length_option;
1164 	int n;
1165 
1166 	for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1167 		len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1168 		      TXADDR_INFO_LENTHG_V1_MAX : remain;
1169 		remain -= len;
1170 
1171 		length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1172 				FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1173 				FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1174 		txaddr_info->length_opt = cpu_to_le16(length_option);
1175 		txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1176 		txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1177 
1178 		dma += len;
1179 		txaddr_info++;
1180 	}
1181 
1182 	WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1183 		  remain, total_len);
1184 
1185 	*add_info_nr = n;
1186 
1187 	return n * sizeof(*txaddr_info);
1188 }
1189 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1190 
1191 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1192 				 struct rtw89_pci_tx_ring *tx_ring,
1193 				 struct rtw89_pci_tx_wd *txwd,
1194 				 struct rtw89_core_tx_request *tx_req)
1195 {
1196 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1197 	const struct rtw89_chip_info *chip = rtwdev->chip;
1198 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1199 	struct rtw89_pci_tx_wp_info *txwp_info;
1200 	void *txaddr_info_addr;
1201 	struct pci_dev *pdev = rtwpci->pdev;
1202 	struct sk_buff *skb = tx_req->skb;
1203 	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1204 	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
1205 	bool en_wd_info = desc_info->en_wd_info;
1206 	u32 txwd_len;
1207 	u32 txwp_len;
1208 	u32 txaddr_info_len;
1209 	dma_addr_t dma;
1210 	int ret;
1211 
1212 	dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1213 	if (dma_mapping_error(&pdev->dev, dma)) {
1214 		rtw89_err(rtwdev, "failed to map skb dma data\n");
1215 		ret = -EBUSY;
1216 		goto err;
1217 	}
1218 
1219 	tx_data->dma = dma;
1220 	rcu_assign_pointer(skb_data->wait, NULL);
1221 
1222 	txwp_len = sizeof(*txwp_info);
1223 	txwd_len = chip->txwd_body_size;
1224 	txwd_len += en_wd_info ? chip->txwd_info_size : 0;
1225 
1226 	txwp_info = txwd->vaddr + txwd_len;
1227 	txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1228 	txwp_info->seq1 = 0;
1229 	txwp_info->seq2 = 0;
1230 	txwp_info->seq3 = 0;
1231 
1232 	tx_ring->tx_cnt++;
1233 	txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1234 	txaddr_info_len =
1235 		rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1236 					    dma, &desc_info->addr_info_nr);
1237 
1238 	txwd->len = txwd_len + txwp_len + txaddr_info_len;
1239 
1240 	rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1241 
1242 	skb_queue_tail(&txwd->queue, skb);
1243 
1244 	return 0;
1245 
1246 err:
1247 	return ret;
1248 }
1249 
1250 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1251 				  struct rtw89_pci_tx_ring *tx_ring,
1252 				  struct rtw89_pci_tx_bd_32 *txbd,
1253 				  struct rtw89_core_tx_request *tx_req)
1254 {
1255 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1256 	const struct rtw89_chip_info *chip = rtwdev->chip;
1257 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1258 	void *txdesc;
1259 	int txdesc_size = chip->h2c_desc_size;
1260 	struct pci_dev *pdev = rtwpci->pdev;
1261 	struct sk_buff *skb = tx_req->skb;
1262 	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1263 	dma_addr_t dma;
1264 
1265 	txdesc = skb_push(skb, txdesc_size);
1266 	memset(txdesc, 0, txdesc_size);
1267 	rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1268 
1269 	dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1270 	if (dma_mapping_error(&pdev->dev, dma)) {
1271 		rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1272 		return -EBUSY;
1273 	}
1274 
1275 	tx_data->dma = dma;
1276 	txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1277 	txbd->length = cpu_to_le16(skb->len);
1278 	txbd->dma = cpu_to_le32(tx_data->dma);
1279 	skb_queue_tail(&rtwpci->h2c_queue, skb);
1280 
1281 	rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1282 
1283 	return 0;
1284 }
1285 
1286 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1287 				 struct rtw89_pci_tx_ring *tx_ring,
1288 				 struct rtw89_pci_tx_bd_32 *txbd,
1289 				 struct rtw89_core_tx_request *tx_req)
1290 {
1291 	struct rtw89_pci_tx_wd *txwd;
1292 	int ret;
1293 
1294 	/* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1295 	 * buffer with WD BODY only. So here we don't need to check the free
1296 	 * pages of the wd ring.
1297 	 */
1298 	if (tx_ring->txch == RTW89_TXCH_CH12)
1299 		return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1300 
1301 	txwd = rtw89_pci_dequeue_txwd(tx_ring);
1302 	if (!txwd) {
1303 		rtw89_err(rtwdev, "no available TXWD\n");
1304 		ret = -ENOSPC;
1305 		goto err;
1306 	}
1307 
1308 	ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1309 	if (ret) {
1310 		rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1311 		goto err_enqueue_wd;
1312 	}
1313 
1314 	list_add_tail(&txwd->list, &tx_ring->busy_pages);
1315 
1316 	txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1317 	txbd->length = cpu_to_le16(txwd->len);
1318 	txbd->dma = cpu_to_le32(txwd->paddr);
1319 
1320 	rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1321 
1322 	return 0;
1323 
1324 err_enqueue_wd:
1325 	rtw89_pci_enqueue_txwd(tx_ring, txwd);
1326 err:
1327 	return ret;
1328 }
1329 
1330 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1331 			      u8 txch)
1332 {
1333 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1334 	struct rtw89_pci_tx_ring *tx_ring;
1335 	struct rtw89_pci_tx_bd_32 *txbd;
1336 	u32 n_avail_txbd;
1337 	int ret = 0;
1338 
1339 	/* check the tx type and dma channel for fw cmd queue */
1340 	if ((txch == RTW89_TXCH_CH12 ||
1341 	     tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1342 	    (txch != RTW89_TXCH_CH12 ||
1343 	     tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1344 		rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1345 		return -EINVAL;
1346 	}
1347 
1348 	tx_ring = &rtwpci->tx_rings[txch];
1349 	spin_lock_bh(&rtwpci->trx_lock);
1350 
1351 	n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring);
1352 	if (n_avail_txbd == 0) {
1353 		rtw89_err(rtwdev, "no available TXBD\n");
1354 		ret = -ENOSPC;
1355 		goto err_unlock;
1356 	}
1357 
1358 	txbd = rtw89_pci_get_next_txbd(tx_ring);
1359 	ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1360 	if (ret) {
1361 		rtw89_err(rtwdev, "failed to submit TXBD\n");
1362 		goto err_unlock;
1363 	}
1364 
1365 	spin_unlock_bh(&rtwpci->trx_lock);
1366 	return 0;
1367 
1368 err_unlock:
1369 	spin_unlock_bh(&rtwpci->trx_lock);
1370 	return ret;
1371 }
1372 
1373 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1374 {
1375 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1376 	int ret;
1377 
1378 	ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1379 	if (ret) {
1380 		rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1381 		return ret;
1382 	}
1383 
1384 	return 0;
1385 }
1386 
1387 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = {
1388 	[RTW89_TXCH_ACH0] = {.start_idx = 0,  .max_num = 5, .min_num = 2},
1389 	[RTW89_TXCH_ACH1] = {.start_idx = 5,  .max_num = 5, .min_num = 2},
1390 	[RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1391 	[RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1392 	[RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1393 	[RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1394 	[RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1395 	[RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1396 	[RTW89_TXCH_CH8]  = {.start_idx = 40, .max_num = 5, .min_num = 1},
1397 	[RTW89_TXCH_CH9]  = {.start_idx = 45, .max_num = 5, .min_num = 1},
1398 	[RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1399 	[RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1400 	[RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1401 };
1402 EXPORT_SYMBOL(rtw89_bd_ram_table_dual);
1403 
1404 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = {
1405 	[RTW89_TXCH_ACH0] = {.start_idx = 0,  .max_num = 5, .min_num = 2},
1406 	[RTW89_TXCH_ACH1] = {.start_idx = 5,  .max_num = 5, .min_num = 2},
1407 	[RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1408 	[RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1409 	[RTW89_TXCH_CH8]  = {.start_idx = 20, .max_num = 4, .min_num = 1},
1410 	[RTW89_TXCH_CH9]  = {.start_idx = 24, .max_num = 4, .min_num = 1},
1411 	[RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1},
1412 };
1413 EXPORT_SYMBOL(rtw89_bd_ram_table_single);
1414 
1415 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1416 {
1417 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1418 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1419 	const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table;
1420 	struct rtw89_pci_tx_ring *tx_ring;
1421 	struct rtw89_pci_rx_ring *rx_ring;
1422 	struct rtw89_pci_dma_ring *bd_ring;
1423 	const struct rtw89_pci_bd_ram *bd_ram;
1424 	u32 addr_num;
1425 	u32 addr_bdram;
1426 	u32 addr_desa_l;
1427 	u32 val32;
1428 	int i;
1429 
1430 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1431 		if (info->tx_dma_ch_mask & BIT(i))
1432 			continue;
1433 
1434 		tx_ring = &rtwpci->tx_rings[i];
1435 		bd_ring = &tx_ring->bd_ring;
1436 		bd_ram = &bd_ram_table[i];
1437 		addr_num = bd_ring->addr.num;
1438 		addr_bdram = bd_ring->addr.bdram;
1439 		addr_desa_l = bd_ring->addr.desa_l;
1440 		bd_ring->wp = 0;
1441 		bd_ring->rp = 0;
1442 
1443 		val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1444 			FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1445 			FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1446 
1447 		rtw89_write16(rtwdev, addr_num, bd_ring->len);
1448 		rtw89_write32(rtwdev, addr_bdram, val32);
1449 		rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1450 	}
1451 
1452 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
1453 		rx_ring = &rtwpci->rx_rings[i];
1454 		bd_ring = &rx_ring->bd_ring;
1455 		addr_num = bd_ring->addr.num;
1456 		addr_desa_l = bd_ring->addr.desa_l;
1457 		bd_ring->wp = 0;
1458 		bd_ring->rp = 0;
1459 		rx_ring->diliver_skb = NULL;
1460 		rx_ring->diliver_desc.ready = false;
1461 
1462 		rtw89_write16(rtwdev, addr_num, bd_ring->len);
1463 		rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1464 	}
1465 }
1466 
1467 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1468 				      struct rtw89_pci_tx_ring *tx_ring)
1469 {
1470 	rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1471 	rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1472 }
1473 
1474 static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1475 {
1476 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1477 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1478 	int txch;
1479 
1480 	rtw89_pci_reset_trx_rings(rtwdev);
1481 
1482 	spin_lock_bh(&rtwpci->trx_lock);
1483 	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1484 		if (info->tx_dma_ch_mask & BIT(txch))
1485 			continue;
1486 		if (txch == RTW89_TXCH_CH12) {
1487 			rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1488 						skb_queue_len(&rtwpci->h2c_queue), true);
1489 			continue;
1490 		}
1491 		rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1492 	}
1493 	spin_unlock_bh(&rtwpci->trx_lock);
1494 }
1495 
1496 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1497 {
1498 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1499 	unsigned long flags;
1500 
1501 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
1502 	rtwpci->running = true;
1503 	rtw89_chip_enable_intr(rtwdev, rtwpci);
1504 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1505 }
1506 
1507 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1508 {
1509 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1510 	unsigned long flags;
1511 
1512 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
1513 	rtwpci->running = false;
1514 	rtw89_chip_disable_intr(rtwdev, rtwpci);
1515 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1516 }
1517 
1518 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1519 {
1520 	rtw89_core_napi_start(rtwdev);
1521 	rtw89_pci_enable_intr_lock(rtwdev);
1522 
1523 	return 0;
1524 }
1525 
1526 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1527 {
1528 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1529 	struct pci_dev *pdev = rtwpci->pdev;
1530 
1531 	rtw89_pci_disable_intr_lock(rtwdev);
1532 	synchronize_irq(pdev->irq);
1533 	rtw89_core_napi_stop(rtwdev);
1534 }
1535 
1536 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1537 {
1538 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1539 	struct pci_dev *pdev = rtwpci->pdev;
1540 
1541 	if (pause) {
1542 		rtw89_pci_disable_intr_lock(rtwdev);
1543 		synchronize_irq(pdev->irq);
1544 		if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1545 			napi_synchronize(&rtwdev->napi);
1546 	} else {
1547 		rtw89_pci_enable_intr_lock(rtwdev);
1548 		rtw89_pci_tx_kick_off_pending(rtwdev);
1549 	}
1550 }
1551 
1552 static
1553 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1554 {
1555 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1556 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1557 	const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1558 	const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1559 	struct rtw89_pci_tx_ring *tx_ring;
1560 	struct rtw89_pci_rx_ring *rx_ring;
1561 	int i;
1562 
1563 	if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1564 		return;
1565 
1566 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1567 		tx_ring = &rtwpci->tx_rings[i];
1568 		tx_ring->bd_ring.addr.idx = low_power ?
1569 					    bd_idx_addr->tx_bd_addrs[i] :
1570 					    dma_addr_set->tx[i].idx;
1571 	}
1572 
1573 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
1574 		rx_ring = &rtwpci->rx_rings[i];
1575 		rx_ring->bd_ring.addr.idx = low_power ?
1576 					    bd_idx_addr->rx_bd_addrs[i] :
1577 					    dma_addr_set->rx[i].idx;
1578 	}
1579 }
1580 
1581 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1582 {
1583 	enum rtw89_pci_intr_mask_cfg cfg;
1584 
1585 	WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1586 
1587 	cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
1588 	rtw89_chip_config_intr_mask(rtwdev, cfg);
1589 	rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1590 }
1591 
1592 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1593 
1594 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1595 {
1596 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1597 	u32 val = readl(rtwpci->mmap + addr);
1598 	int count;
1599 
1600 	for (count = 0; ; count++) {
1601 		if (val != RTW89_R32_DEAD)
1602 			return val;
1603 		if (count >= MAC_REG_POOL_COUNT) {
1604 			rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1605 			return RTW89_R32_DEAD;
1606 		}
1607 		rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1608 		val = readl(rtwpci->mmap + addr);
1609 	}
1610 
1611 	return val;
1612 }
1613 
1614 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1615 {
1616 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1617 	u32 addr32, val32, shift;
1618 
1619 	if (!ACCESS_CMAC(addr))
1620 		return readb(rtwpci->mmap + addr);
1621 
1622 	addr32 = addr & ~0x3;
1623 	shift = (addr & 0x3) * 8;
1624 	val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1625 	return val32 >> shift;
1626 }
1627 
1628 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1629 {
1630 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1631 	u32 addr32, val32, shift;
1632 
1633 	if (!ACCESS_CMAC(addr))
1634 		return readw(rtwpci->mmap + addr);
1635 
1636 	addr32 = addr & ~0x3;
1637 	shift = (addr & 0x3) * 8;
1638 	val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1639 	return val32 >> shift;
1640 }
1641 
1642 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1643 {
1644 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1645 
1646 	if (!ACCESS_CMAC(addr))
1647 		return readl(rtwpci->mmap + addr);
1648 
1649 	return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1650 }
1651 
1652 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1653 {
1654 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1655 
1656 	writeb(data, rtwpci->mmap + addr);
1657 }
1658 
1659 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1660 {
1661 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1662 
1663 	writew(data, rtwpci->mmap + addr);
1664 }
1665 
1666 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1667 {
1668 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1669 
1670 	writel(data, rtwpci->mmap + addr);
1671 }
1672 
1673 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1674 {
1675 	const struct rtw89_pci_info *info = rtwdev->pci_info;
1676 
1677 	if (enable)
1678 		rtw89_write32_set(rtwdev, info->init_cfg_reg,
1679 				  info->rxhci_en_bit | info->txhci_en_bit);
1680 	else
1681 		rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1682 				  info->rxhci_en_bit | info->txhci_en_bit);
1683 }
1684 
1685 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1686 {
1687 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1688 	u32 reg, mask;
1689 
1690 	if (chip_id == RTL8852C) {
1691 		reg = R_AX_HAXI_INIT_CFG1;
1692 		mask = B_AX_STOP_AXI_MST;
1693 	} else {
1694 		reg = R_AX_PCIE_DMA_STOP1;
1695 		mask = B_AX_STOP_PCIEIO;
1696 	}
1697 
1698 	if (enable)
1699 		rtw89_write32_clr(rtwdev, reg, mask);
1700 	else
1701 		rtw89_write32_set(rtwdev, reg, mask);
1702 }
1703 
1704 static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1705 {
1706 	rtw89_pci_ctrl_dma_io(rtwdev, enable);
1707 	rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1708 }
1709 
1710 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1711 {
1712 	u16 val;
1713 
1714 	rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1715 
1716 	val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1717 	switch (speed) {
1718 	case PCIE_PHY_GEN1:
1719 		if (addr < 0x20)
1720 			val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
1721 		else
1722 			val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
1723 		break;
1724 	case PCIE_PHY_GEN2:
1725 		if (addr < 0x20)
1726 			val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
1727 		else
1728 			val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
1729 		break;
1730 	default:
1731 		rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
1732 		return -EINVAL;
1733 	}
1734 	rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
1735 	rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
1736 
1737 	return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
1738 				 false, rtwdev, R_AX_MDIO_CFG);
1739 }
1740 
1741 static int
1742 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
1743 {
1744 	int ret;
1745 
1746 	ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
1747 	if (ret) {
1748 		rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
1749 		return ret;
1750 	}
1751 	*val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
1752 
1753 	return 0;
1754 }
1755 
1756 static int
1757 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
1758 {
1759 	int ret;
1760 
1761 	rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
1762 	ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
1763 	if (ret) {
1764 		rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
1765 		return ret;
1766 	}
1767 
1768 	return 0;
1769 }
1770 
1771 static int
1772 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
1773 {
1774 	u32 shift;
1775 	int ret;
1776 	u16 val;
1777 
1778 	ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1779 	if (ret)
1780 		return ret;
1781 
1782 	shift = __ffs(mask);
1783 	val &= ~mask;
1784 	val |= ((data << shift) & mask);
1785 
1786 	ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
1787 	if (ret)
1788 		return ret;
1789 
1790 	return 0;
1791 }
1792 
1793 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1794 {
1795 	int ret;
1796 	u16 val;
1797 
1798 	ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1799 	if (ret)
1800 		return ret;
1801 	ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
1802 	if (ret)
1803 		return ret;
1804 
1805 	return 0;
1806 }
1807 
1808 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1809 {
1810 	int ret;
1811 	u16 val;
1812 
1813 	ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1814 	if (ret)
1815 		return ret;
1816 	ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
1817 	if (ret)
1818 		return ret;
1819 
1820 	return 0;
1821 }
1822 
1823 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1824 				       u8 data)
1825 {
1826 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1827 	struct pci_dev *pdev = rtwpci->pdev;
1828 
1829 	return pci_write_config_byte(pdev, addr, data);
1830 }
1831 
1832 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1833 				      u8 *value)
1834 {
1835 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1836 	struct pci_dev *pdev = rtwpci->pdev;
1837 
1838 	return pci_read_config_byte(pdev, addr, value);
1839 }
1840 
1841 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
1842 				     u8 bit)
1843 {
1844 	u8 value;
1845 	int ret;
1846 
1847 	ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1848 	if (ret)
1849 		return ret;
1850 
1851 	value |= bit;
1852 	ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1853 
1854 	return ret;
1855 }
1856 
1857 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
1858 				     u8 bit)
1859 {
1860 	u8 value;
1861 	int ret;
1862 
1863 	ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1864 	if (ret)
1865 		return ret;
1866 
1867 	value &= ~bit;
1868 	ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1869 
1870 	return ret;
1871 }
1872 
1873 static int
1874 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
1875 {
1876 	u16 val, tar;
1877 	int ret;
1878 
1879 	/* Enable counter */
1880 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
1881 	if (ret)
1882 		return ret;
1883 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1884 				 phy_rate);
1885 	if (ret)
1886 		return ret;
1887 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
1888 				 phy_rate);
1889 	if (ret)
1890 		return ret;
1891 
1892 	fsleep(300);
1893 
1894 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
1895 	if (ret)
1896 		return ret;
1897 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1898 				 phy_rate);
1899 	if (ret)
1900 		return ret;
1901 
1902 	tar = tar & 0x0FFF;
1903 	if (tar == 0 || tar == 0x0FFF) {
1904 		rtw89_err(rtwdev, "[ERR]Get target failed.\n");
1905 		return -EINVAL;
1906 	}
1907 
1908 	*target = tar;
1909 
1910 	return 0;
1911 }
1912 
1913 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
1914 {
1915 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1916 	int ret;
1917 
1918 	if (chip_id != RTL8852B && chip_id != RTL8851B)
1919 		return 0;
1920 
1921 	ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
1922 				      PCIE_AUTOK_4, PCIE_PHY_GEN1);
1923 	return ret;
1924 }
1925 
1926 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
1927 {
1928 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
1929 	enum rtw89_pcie_phy phy_rate;
1930 	u16 val16, mgn_set, div_set, tar;
1931 	u8 val8, bdr_ori;
1932 	bool l1_flag = false;
1933 	int ret = 0;
1934 
1935 	if (chip_id != RTL8852B && chip_id != RTL8851B)
1936 		return 0;
1937 
1938 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
1939 	if (ret) {
1940 		rtw89_err(rtwdev, "[ERR]pci config read %X\n",
1941 			  RTW89_PCIE_PHY_RATE);
1942 		return ret;
1943 	}
1944 
1945 	if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
1946 		phy_rate = PCIE_PHY_GEN1;
1947 	} else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
1948 		phy_rate = PCIE_PHY_GEN2;
1949 	} else {
1950 		rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
1951 		return -EOPNOTSUPP;
1952 	}
1953 	/* Disable L1BD */
1954 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
1955 	if (ret) {
1956 		rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
1957 		return ret;
1958 	}
1959 
1960 	if (bdr_ori & RTW89_PCIE_BIT_L1) {
1961 		ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
1962 						  bdr_ori & ~RTW89_PCIE_BIT_L1);
1963 		if (ret) {
1964 			rtw89_err(rtwdev, "[ERR]pci config write %X\n",
1965 				  RTW89_PCIE_L1_CTRL);
1966 			return ret;
1967 		}
1968 		l1_flag = true;
1969 	}
1970 
1971 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
1972 	if (ret) {
1973 		rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
1974 		goto end;
1975 	}
1976 
1977 	if (val16 & B_AX_CALIB_EN) {
1978 		ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
1979 					 val16 & ~B_AX_CALIB_EN, phy_rate);
1980 		if (ret) {
1981 			rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1982 			goto end;
1983 		}
1984 	}
1985 
1986 	if (!autook_en)
1987 		goto end;
1988 	/* Set div */
1989 	ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
1990 	if (ret) {
1991 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
1992 		goto end;
1993 	}
1994 
1995 	/* Obtain div and margin */
1996 	ret = __get_target(rtwdev, &tar, phy_rate);
1997 	if (ret) {
1998 		rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
1999 		goto end;
2000 	}
2001 
2002 	mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
2003 
2004 	if (mgn_set >= 128) {
2005 		div_set = 0x0003;
2006 		mgn_set = 0x000F;
2007 	} else if (mgn_set >= 64) {
2008 		div_set = 0x0003;
2009 		mgn_set >>= 3;
2010 	} else if (mgn_set >= 32) {
2011 		div_set = 0x0002;
2012 		mgn_set >>= 2;
2013 	} else if (mgn_set >= 16) {
2014 		div_set = 0x0001;
2015 		mgn_set >>= 1;
2016 	} else if (mgn_set == 0) {
2017 		rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2018 		goto end;
2019 	} else {
2020 		div_set = 0x0000;
2021 	}
2022 
2023 	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2024 	if (ret) {
2025 		rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2026 		goto end;
2027 	}
2028 
2029 	val16 |= u16_encode_bits(div_set, B_AX_DIV);
2030 
2031 	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2032 	if (ret) {
2033 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2034 		goto end;
2035 	}
2036 
2037 	ret = __get_target(rtwdev, &tar, phy_rate);
2038 	if (ret) {
2039 		rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2040 		goto end;
2041 	}
2042 
2043 	rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2044 		    tar, div_set, mgn_set);
2045 	ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2046 				 (tar & 0x0FFF) | (mgn_set << 12), phy_rate);
2047 	if (ret) {
2048 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2049 		goto end;
2050 	}
2051 
2052 	/* Enable function */
2053 	ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2054 	if (ret) {
2055 		rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2056 		goto end;
2057 	}
2058 
2059 	/* CLK delay = 0 */
2060 	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2061 					  PCIE_CLKDLY_HW_0);
2062 
2063 end:
2064 	/* Set L1BD to ori */
2065 	if (l1_flag) {
2066 		ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2067 						  bdr_ori);
2068 		if (ret) {
2069 			rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2070 				  RTW89_PCIE_L1_CTRL);
2071 			return ret;
2072 		}
2073 	}
2074 
2075 	return ret;
2076 }
2077 
2078 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2079 {
2080 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2081 	int ret;
2082 
2083 	if (chip_id == RTL8852A) {
2084 		ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2085 					     PCIE_PHY_GEN1);
2086 		if (ret)
2087 			return ret;
2088 		ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2089 					     PCIE_PHY_GEN2);
2090 		if (ret)
2091 			return ret;
2092 	} else if (chip_id == RTL8852C) {
2093 		rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2094 				  B_AX_DEGLITCH);
2095 		rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2096 				  B_AX_DEGLITCH);
2097 	}
2098 
2099 	return 0;
2100 }
2101 
2102 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2103 {
2104 	if (rtwdev->chip->chip_id != RTL8852A)
2105 		return;
2106 
2107 	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2108 }
2109 
2110 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2111 {
2112 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2113 
2114 	if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
2115 		return;
2116 
2117 	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2118 }
2119 
2120 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2121 {
2122 	int ret;
2123 
2124 	if (rtwdev->chip->chip_id != RTL8852A)
2125 		return 0;
2126 
2127 	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2128 				     PCIE_PHY_GEN1);
2129 	if (ret)
2130 		return ret;
2131 
2132 	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2133 				     PCIE_PHY_GEN2);
2134 	if (ret)
2135 		return ret;
2136 
2137 	return 0;
2138 }
2139 
2140 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2141 {
2142 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2143 
2144 	if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
2145 		return;
2146 
2147 	rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2148 }
2149 
2150 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2151 {
2152 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2153 
2154 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
2155 		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2156 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2157 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2158 				  B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2159 	} else if (rtwdev->chip->chip_id == RTL8852C) {
2160 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2161 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2162 	}
2163 }
2164 
2165 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2166 {
2167 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2168 
2169 	if (chip_id != RTL8852B && chip_id != RTL8851B)
2170 		return 0;
2171 
2172 	return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2173 				       PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1);
2174 }
2175 
2176 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)
2177 {
2178 	if (pwr_up)
2179 		rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2180 	else
2181 		rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2182 }
2183 
2184 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2185 {
2186 	if (rtwdev->chip->chip_id != RTL8852C)
2187 		return;
2188 
2189 	rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2190 	rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2191 }
2192 
2193 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2194 {
2195 	if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2196 		return;
2197 
2198 	rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2199 }
2200 
2201 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2202 {
2203 	if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2204 		return;
2205 
2206 	rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2207 			  B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2208 	rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2209 	rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2210 			  B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2211 }
2212 
2213 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2214 {
2215 	if (rtwdev->chip->chip_id != RTL8852C)
2216 		return;
2217 
2218 	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2219 }
2220 
2221 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2222 {
2223 	if (rtwdev->chip->chip_id != RTL8852C)
2224 		return;
2225 
2226 	rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2227 }
2228 
2229 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2230 {
2231 	if (rtwdev->chip->chip_id == RTL8852C)
2232 		return;
2233 
2234 	rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2235 			  B_AX_SIC_EN_FORCE_CLKREQ);
2236 }
2237 
2238 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2239 {
2240 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2241 	u32 lbc;
2242 
2243 	if (rtwdev->chip->chip_id == RTL8852C)
2244 		return;
2245 
2246 	lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2247 	if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2248 		lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER);
2249 		lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2250 		rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2251 	} else {
2252 		lbc &= ~B_AX_LBC_EN;
2253 	}
2254 	rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2255 }
2256 
2257 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2258 {
2259 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2260 	u32 val32;
2261 
2262 	if (rtwdev->chip->chip_id != RTL8852C)
2263 		return;
2264 
2265 	if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2266 		val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2267 				   info->io_rcy_tmr);
2268 		rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2269 		rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2270 		rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2271 
2272 		rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2273 		rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2274 		rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2275 	} else {
2276 		rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2277 		rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2278 		rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2279 	}
2280 
2281 	rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2282 }
2283 
2284 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2285 {
2286 	if (rtwdev->chip->chip_id == RTL8852C)
2287 		return;
2288 
2289 	rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2290 			  B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2291 
2292 	if (rtwdev->chip->chip_id == RTL8852A)
2293 		rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2294 				  B_AX_EN_CHKDSC_NO_RX_STUCK);
2295 }
2296 
2297 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2298 {
2299 	if (rtwdev->chip->chip_id == RTL8852C)
2300 		return;
2301 
2302 	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2303 			  B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2304 }
2305 
2306 static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev)
2307 {
2308 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2309 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2310 	u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2311 		  B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2312 		  B_AX_CLR_CH12_IDX;
2313 	u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2314 	u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2315 
2316 	if (chip_id == RTL8852A || chip_id == RTL8852C)
2317 		val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2318 		       B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2319 	/* clear DMA indexes */
2320 	rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2321 	if (chip_id == RTL8852A || chip_id == RTL8852C)
2322 		rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2323 				  B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2324 	rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2325 			  B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2326 }
2327 
2328 static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2329 {
2330 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2331 	u32 ret, check, dma_busy;
2332 	u32 dma_busy1 = info->dma_busy1.addr;
2333 	u32 dma_busy2 = info->dma_busy2_reg;
2334 
2335 	check = info->dma_busy1.mask;
2336 
2337 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2338 				10, 100, false, rtwdev, dma_busy1);
2339 	if (ret)
2340 		return ret;
2341 
2342 	if (!dma_busy2)
2343 		return 0;
2344 
2345 	check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2346 
2347 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2348 				10, 100, false, rtwdev, dma_busy2);
2349 	if (ret)
2350 		return ret;
2351 
2352 	return 0;
2353 }
2354 
2355 static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2356 {
2357 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2358 	u32 ret, check, dma_busy;
2359 	u32 dma_busy3 = info->dma_busy3_reg;
2360 
2361 	check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2362 
2363 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2364 				10, 100, false, rtwdev, dma_busy3);
2365 	if (ret)
2366 		return ret;
2367 
2368 	return 0;
2369 }
2370 
2371 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2372 {
2373 	u32 ret;
2374 
2375 	ret = rtw89_poll_txdma_ch_idle_pcie(rtwdev);
2376 	if (ret) {
2377 		rtw89_err(rtwdev, "txdma ch busy\n");
2378 		return ret;
2379 	}
2380 
2381 	ret = rtw89_poll_rxdma_ch_idle_pcie(rtwdev);
2382 	if (ret) {
2383 		rtw89_err(rtwdev, "rxdma ch busy\n");
2384 		return ret;
2385 	}
2386 
2387 	return 0;
2388 }
2389 
2390 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2391 {
2392 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2393 	enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
2394 	enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
2395 	enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
2396 	enum mac_ax_tag_mode tag_mode = info->tag_mode;
2397 	enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
2398 	enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
2399 	enum mac_ax_tx_burst tx_burst = info->tx_burst;
2400 	enum mac_ax_rx_burst rx_burst = info->rx_burst;
2401 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2402 	u8 cv = rtwdev->hal.cv;
2403 	u32 val32;
2404 
2405 	if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2406 		if (chip_id == RTL8852A && cv == CHIP_CBV)
2407 			rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2408 	} else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2409 		if (chip_id == RTL8852A || chip_id == RTL8852B)
2410 			rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2411 	}
2412 
2413 	if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
2414 		if (chip_id == RTL8852A && cv == CHIP_CBV)
2415 			rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2416 	} else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
2417 		if (chip_id == RTL8852A || chip_id == RTL8852B)
2418 			rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2419 	}
2420 
2421 	if (rxbd_mode == MAC_AX_RXBD_PKT) {
2422 		rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2423 	} else if (rxbd_mode == MAC_AX_RXBD_SEP) {
2424 		rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2425 
2426 		if (chip_id == RTL8852A || chip_id == RTL8852B)
2427 			rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2428 					   B_AX_PCIE_RX_APPLEN_MASK, 0);
2429 	}
2430 
2431 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2432 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2433 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2434 	} else if (chip_id == RTL8852C) {
2435 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2436 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2437 	}
2438 
2439 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2440 		if (tag_mode == MAC_AX_TAG_SGL) {
2441 			val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2442 					    ~B_AX_LATENCY_CONTROL;
2443 			rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2444 		} else if (tag_mode == MAC_AX_TAG_MULTI) {
2445 			val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2446 					    B_AX_LATENCY_CONTROL;
2447 			rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2448 		}
2449 	}
2450 
2451 	rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2452 			   info->multi_tag_num);
2453 
2454 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2455 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2456 				   wd_dma_idle_intvl);
2457 		rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2458 				   wd_dma_act_intvl);
2459 	} else if (chip_id == RTL8852C) {
2460 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2461 				   wd_dma_idle_intvl);
2462 		rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2463 				   wd_dma_act_intvl);
2464 	}
2465 
2466 	if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2467 		rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2468 				  B_AX_HOST_ADDR_INFO_8B_SEL);
2469 		rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2470 	} else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2471 		rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2472 				  B_AX_HOST_ADDR_INFO_8B_SEL);
2473 		rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2474 	}
2475 
2476 	return 0;
2477 }
2478 
2479 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2480 {
2481 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2482 
2483 	if (rtwdev->chip->chip_id == RTL8852A) {
2484 		/* ltr sw trigger */
2485 		rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2486 	}
2487 	info->ltr_set(rtwdev, false);
2488 	rtw89_pci_ctrl_dma_all(rtwdev, false);
2489 	rtw89_pci_clr_idx_all(rtwdev);
2490 
2491 	return 0;
2492 }
2493 
2494 static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
2495 {
2496 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2497 	int ret;
2498 
2499 	rtw89_pci_rxdma_prefth(rtwdev);
2500 	rtw89_pci_l1off_pwroff(rtwdev);
2501 	rtw89_pci_deglitch_setting(rtwdev);
2502 	ret = rtw89_pci_l2_rxen_lat(rtwdev);
2503 	if (ret) {
2504 		rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2505 		return ret;
2506 	}
2507 
2508 	rtw89_pci_aphy_pwrcut(rtwdev);
2509 	rtw89_pci_hci_ldo(rtwdev);
2510 	rtw89_pci_dphy_delay(rtwdev);
2511 
2512 	ret = rtw89_pci_autok_x(rtwdev);
2513 	if (ret) {
2514 		rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2515 		return ret;
2516 	}
2517 
2518 	ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2519 	if (ret) {
2520 		rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2521 		return ret;
2522 	}
2523 
2524 	rtw89_pci_power_wake(rtwdev, true);
2525 	rtw89_pci_autoload_hang(rtwdev);
2526 	rtw89_pci_l12_vmain(rtwdev);
2527 	rtw89_pci_gen2_force_ib(rtwdev);
2528 	rtw89_pci_l1_ent_lat(rtwdev);
2529 	rtw89_pci_wd_exit_l1(rtwdev);
2530 	rtw89_pci_set_sic(rtwdev);
2531 	rtw89_pci_set_lbc(rtwdev);
2532 	rtw89_pci_set_io_rcy(rtwdev);
2533 	rtw89_pci_set_dbg(rtwdev);
2534 	rtw89_pci_set_keep_reg(rtwdev);
2535 
2536 	rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2537 
2538 	/* stop DMA activities */
2539 	rtw89_pci_ctrl_dma_all(rtwdev, false);
2540 
2541 	ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2542 	if (ret) {
2543 		rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2544 		return ret;
2545 	}
2546 
2547 	rtw89_pci_clr_idx_all(rtwdev);
2548 	rtw89_pci_mode_op(rtwdev);
2549 
2550 	/* fill TRX BD indexes */
2551 	rtw89_pci_ops_reset(rtwdev);
2552 
2553 	ret = rtw89_pci_rst_bdram_pcie(rtwdev);
2554 	if (ret) {
2555 		rtw89_warn(rtwdev, "reset bdram busy\n");
2556 		return ret;
2557 	}
2558 
2559 	/* disable all channels except to FW CMD channel to download firmware */
2560 	rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false);
2561 	rtw89_pci_ctrl_txdma_fw_ch_pcie(rtwdev, true);
2562 
2563 	/* start DMA activities */
2564 	rtw89_pci_ctrl_dma_all(rtwdev, true);
2565 
2566 	return 0;
2567 }
2568 
2569 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2570 {
2571 	u32 val;
2572 
2573 	if (!en)
2574 		return 0;
2575 
2576 	val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2577 	if (rtw89_pci_ltr_is_err_reg_val(val))
2578 		return -EINVAL;
2579 	val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2580 	if (rtw89_pci_ltr_is_err_reg_val(val))
2581 		return -EINVAL;
2582 	val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2583 	if (rtw89_pci_ltr_is_err_reg_val(val))
2584 		return -EINVAL;
2585 	val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2586 	if (rtw89_pci_ltr_is_err_reg_val(val))
2587 		return -EINVAL;
2588 
2589 	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2590 						   B_AX_LTR_WD_NOEMP_CHK);
2591 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2592 			   PCI_LTR_SPC_500US);
2593 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2594 			   PCI_LTR_IDLE_TIMER_3_2MS);
2595 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2596 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2597 	rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
2598 	rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
2599 
2600 	return 0;
2601 }
2602 EXPORT_SYMBOL(rtw89_pci_ltr_set);
2603 
2604 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
2605 {
2606 	u32 dec_ctrl;
2607 	u32 val32;
2608 
2609 	val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2610 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2611 		return -EINVAL;
2612 	val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2613 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2614 		return -EINVAL;
2615 	dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
2616 	if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
2617 		return -EINVAL;
2618 	val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
2619 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2620 		return -EINVAL;
2621 	val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
2622 	if (rtw89_pci_ltr_is_err_reg_val(val32))
2623 		return -EINVAL;
2624 
2625 	if (!en) {
2626 		dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
2627 		dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
2628 			    B_AX_LTR_REQ_DRV;
2629 	} else {
2630 		dec_ctrl |= B_AX_LTR_HW_DEC_EN;
2631 	}
2632 
2633 	dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
2634 	dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
2635 
2636 	if (en)
2637 		rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
2638 				  B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
2639 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2640 			   PCI_LTR_IDLE_TIMER_3_2MS);
2641 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2642 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2643 	rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
2644 	rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
2645 	rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
2646 
2647 	return 0;
2648 }
2649 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
2650 
2651 static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)
2652 {
2653 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2654 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2655 	int ret;
2656 
2657 	ret = info->ltr_set(rtwdev, true);
2658 	if (ret) {
2659 		rtw89_err(rtwdev, "pci ltr set fail\n");
2660 		return ret;
2661 	}
2662 	if (chip_id == RTL8852A) {
2663 		/* ltr sw trigger */
2664 		rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
2665 	}
2666 	if (chip_id == RTL8852A || chip_id == RTL8852B) {
2667 		/* ADDR info 8-byte mode */
2668 		rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2669 				  B_AX_HOST_ADDR_INFO_8B_SEL);
2670 		rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2671 	}
2672 
2673 	/* enable DMA for all queues */
2674 	rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true);
2675 
2676 	/* Release PCI IO */
2677 	rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
2678 			  B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
2679 
2680 	return 0;
2681 }
2682 
2683 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
2684 				  struct pci_dev *pdev)
2685 {
2686 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2687 	int ret;
2688 
2689 	ret = pci_enable_device(pdev);
2690 	if (ret) {
2691 		rtw89_err(rtwdev, "failed to enable pci device\n");
2692 		return ret;
2693 	}
2694 
2695 	pci_set_master(pdev);
2696 	pci_set_drvdata(pdev, rtwdev->hw);
2697 
2698 	rtwpci->pdev = pdev;
2699 
2700 	return 0;
2701 }
2702 
2703 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
2704 				     struct pci_dev *pdev)
2705 {
2706 	pci_disable_device(pdev);
2707 }
2708 
2709 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
2710 				   struct pci_dev *pdev)
2711 {
2712 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2713 	unsigned long resource_len;
2714 	u8 bar_id = 2;
2715 	int ret;
2716 
2717 	ret = pci_request_regions(pdev, KBUILD_MODNAME);
2718 	if (ret) {
2719 		rtw89_err(rtwdev, "failed to request pci regions\n");
2720 		goto err;
2721 	}
2722 
2723 	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2724 	if (ret) {
2725 		rtw89_err(rtwdev, "failed to set dma mask to 32-bit\n");
2726 		goto err_release_regions;
2727 	}
2728 
2729 	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
2730 	if (ret) {
2731 		rtw89_err(rtwdev, "failed to set consistent dma mask to 32-bit\n");
2732 		goto err_release_regions;
2733 	}
2734 
2735 	resource_len = pci_resource_len(pdev, bar_id);
2736 	rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len);
2737 	if (!rtwpci->mmap) {
2738 		rtw89_err(rtwdev, "failed to map pci io\n");
2739 		ret = -EIO;
2740 		goto err_release_regions;
2741 	}
2742 
2743 	return 0;
2744 
2745 err_release_regions:
2746 	pci_release_regions(pdev);
2747 err:
2748 	return ret;
2749 }
2750 
2751 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
2752 				    struct pci_dev *pdev)
2753 {
2754 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2755 
2756 	if (rtwpci->mmap) {
2757 		pci_iounmap(pdev, rtwpci->mmap);
2758 		pci_release_regions(pdev);
2759 	}
2760 }
2761 
2762 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
2763 				      struct pci_dev *pdev,
2764 				      struct rtw89_pci_tx_ring *tx_ring)
2765 {
2766 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2767 	u8 *head = wd_ring->head;
2768 	dma_addr_t dma = wd_ring->dma;
2769 	u32 page_size = wd_ring->page_size;
2770 	u32 page_num = wd_ring->page_num;
2771 	u32 ring_sz = page_size * page_num;
2772 
2773 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2774 	wd_ring->head = NULL;
2775 }
2776 
2777 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
2778 				   struct pci_dev *pdev,
2779 				   struct rtw89_pci_tx_ring *tx_ring)
2780 {
2781 	int ring_sz;
2782 	u8 *head;
2783 	dma_addr_t dma;
2784 
2785 	head = tx_ring->bd_ring.head;
2786 	dma = tx_ring->bd_ring.dma;
2787 	ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len;
2788 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2789 
2790 	tx_ring->bd_ring.head = NULL;
2791 }
2792 
2793 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
2794 				    struct pci_dev *pdev)
2795 {
2796 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2797 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2798 	struct rtw89_pci_tx_ring *tx_ring;
2799 	int i;
2800 
2801 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
2802 		if (info->tx_dma_ch_mask & BIT(i))
2803 			continue;
2804 		tx_ring = &rtwpci->tx_rings[i];
2805 		rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2806 		rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
2807 	}
2808 }
2809 
2810 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
2811 				   struct pci_dev *pdev,
2812 				   struct rtw89_pci_rx_ring *rx_ring)
2813 {
2814 	struct rtw89_pci_rx_info *rx_info;
2815 	struct sk_buff *skb;
2816 	dma_addr_t dma;
2817 	u32 buf_sz;
2818 	u8 *head;
2819 	int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len;
2820 	int i;
2821 
2822 	buf_sz = rx_ring->buf_sz;
2823 	for (i = 0; i < rx_ring->bd_ring.len; i++) {
2824 		skb = rx_ring->buf[i];
2825 		if (!skb)
2826 			continue;
2827 
2828 		rx_info = RTW89_PCI_RX_SKB_CB(skb);
2829 		dma = rx_info->dma;
2830 		dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
2831 		dev_kfree_skb(skb);
2832 		rx_ring->buf[i] = NULL;
2833 	}
2834 
2835 	head = rx_ring->bd_ring.head;
2836 	dma = rx_ring->bd_ring.dma;
2837 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2838 
2839 	rx_ring->bd_ring.head = NULL;
2840 }
2841 
2842 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
2843 				    struct pci_dev *pdev)
2844 {
2845 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2846 	struct rtw89_pci_rx_ring *rx_ring;
2847 	int i;
2848 
2849 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
2850 		rx_ring = &rtwpci->rx_rings[i];
2851 		rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
2852 	}
2853 }
2854 
2855 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
2856 				     struct pci_dev *pdev)
2857 {
2858 	rtw89_pci_free_rx_rings(rtwdev, pdev);
2859 	rtw89_pci_free_tx_rings(rtwdev, pdev);
2860 }
2861 
2862 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
2863 				struct rtw89_pci_rx_ring *rx_ring,
2864 				struct sk_buff *skb, int buf_sz, u32 idx)
2865 {
2866 	struct rtw89_pci_rx_info *rx_info;
2867 	struct rtw89_pci_rx_bd_32 *rx_bd;
2868 	dma_addr_t dma;
2869 
2870 	if (!skb)
2871 		return -EINVAL;
2872 
2873 	dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
2874 	if (dma_mapping_error(&pdev->dev, dma))
2875 		return -EBUSY;
2876 
2877 	rx_info = RTW89_PCI_RX_SKB_CB(skb);
2878 	rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
2879 
2880 	memset(rx_bd, 0, sizeof(*rx_bd));
2881 	rx_bd->buf_size = cpu_to_le16(buf_sz);
2882 	rx_bd->dma = cpu_to_le32(dma);
2883 	rx_info->dma = dma;
2884 
2885 	return 0;
2886 }
2887 
2888 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
2889 				      struct pci_dev *pdev,
2890 				      struct rtw89_pci_tx_ring *tx_ring,
2891 				      enum rtw89_tx_channel txch)
2892 {
2893 	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2894 	struct rtw89_pci_tx_wd *txwd;
2895 	dma_addr_t dma;
2896 	dma_addr_t cur_paddr;
2897 	u8 *head;
2898 	u8 *cur_vaddr;
2899 	u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
2900 	u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
2901 	u32 ring_sz = page_size * page_num;
2902 	u32 page_offset;
2903 	int i;
2904 
2905 	/* FWCMD queue doesn't use txwd as pages */
2906 	if (txch == RTW89_TXCH_CH12)
2907 		return 0;
2908 
2909 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2910 	if (!head)
2911 		return -ENOMEM;
2912 
2913 	INIT_LIST_HEAD(&wd_ring->free_pages);
2914 	wd_ring->head = head;
2915 	wd_ring->dma = dma;
2916 	wd_ring->page_size = page_size;
2917 	wd_ring->page_num = page_num;
2918 
2919 	page_offset = 0;
2920 	for (i = 0; i < page_num; i++) {
2921 		txwd = &wd_ring->pages[i];
2922 		cur_paddr = dma + page_offset;
2923 		cur_vaddr = head + page_offset;
2924 
2925 		skb_queue_head_init(&txwd->queue);
2926 		INIT_LIST_HEAD(&txwd->list);
2927 		txwd->paddr = cur_paddr;
2928 		txwd->vaddr = cur_vaddr;
2929 		txwd->len = page_size;
2930 		txwd->seq = i;
2931 		rtw89_pci_enqueue_txwd(tx_ring, txwd);
2932 
2933 		page_offset += page_size;
2934 	}
2935 
2936 	return 0;
2937 }
2938 
2939 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
2940 				   struct pci_dev *pdev,
2941 				   struct rtw89_pci_tx_ring *tx_ring,
2942 				   u32 desc_size, u32 len,
2943 				   enum rtw89_tx_channel txch)
2944 {
2945 	const struct rtw89_pci_ch_dma_addr *txch_addr;
2946 	int ring_sz = desc_size * len;
2947 	u8 *head;
2948 	dma_addr_t dma;
2949 	int ret;
2950 
2951 	ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
2952 	if (ret) {
2953 		rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
2954 		goto err;
2955 	}
2956 
2957 	ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
2958 	if (ret) {
2959 		rtw89_err(rtwdev, "failed to get address of txch %d", txch);
2960 		goto err_free_wd_ring;
2961 	}
2962 
2963 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2964 	if (!head) {
2965 		ret = -ENOMEM;
2966 		goto err_free_wd_ring;
2967 	}
2968 
2969 	INIT_LIST_HEAD(&tx_ring->busy_pages);
2970 	tx_ring->bd_ring.head = head;
2971 	tx_ring->bd_ring.dma = dma;
2972 	tx_ring->bd_ring.len = len;
2973 	tx_ring->bd_ring.desc_size = desc_size;
2974 	tx_ring->bd_ring.addr = *txch_addr;
2975 	tx_ring->bd_ring.wp = 0;
2976 	tx_ring->bd_ring.rp = 0;
2977 	tx_ring->txch = txch;
2978 
2979 	return 0;
2980 
2981 err_free_wd_ring:
2982 	rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2983 err:
2984 	return ret;
2985 }
2986 
2987 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
2988 				    struct pci_dev *pdev)
2989 {
2990 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2991 	const struct rtw89_pci_info *info = rtwdev->pci_info;
2992 	struct rtw89_pci_tx_ring *tx_ring;
2993 	u32 desc_size;
2994 	u32 len;
2995 	u32 i, tx_allocated;
2996 	int ret;
2997 
2998 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
2999 		if (info->tx_dma_ch_mask & BIT(i))
3000 			continue;
3001 		tx_ring = &rtwpci->tx_rings[i];
3002 		desc_size = sizeof(struct rtw89_pci_tx_bd_32);
3003 		len = RTW89_PCI_TXBD_NUM_MAX;
3004 		ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3005 					      desc_size, len, i);
3006 		if (ret) {
3007 			rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3008 			goto err_free;
3009 		}
3010 	}
3011 
3012 	return 0;
3013 
3014 err_free:
3015 	tx_allocated = i;
3016 	for (i = 0; i < tx_allocated; i++) {
3017 		tx_ring = &rtwpci->tx_rings[i];
3018 		rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3019 	}
3020 
3021 	return ret;
3022 }
3023 
3024 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3025 				   struct pci_dev *pdev,
3026 				   struct rtw89_pci_rx_ring *rx_ring,
3027 				   u32 desc_size, u32 len, u32 rxch)
3028 {
3029 	const struct rtw89_pci_ch_dma_addr *rxch_addr;
3030 	struct sk_buff *skb;
3031 	u8 *head;
3032 	dma_addr_t dma;
3033 	int ring_sz = desc_size * len;
3034 	int buf_sz = RTW89_PCI_RX_BUF_SIZE;
3035 	int i, allocated;
3036 	int ret;
3037 
3038 	ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3039 	if (ret) {
3040 		rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3041 		return ret;
3042 	}
3043 
3044 	head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3045 	if (!head) {
3046 		ret = -ENOMEM;
3047 		goto err;
3048 	}
3049 
3050 	rx_ring->bd_ring.head = head;
3051 	rx_ring->bd_ring.dma = dma;
3052 	rx_ring->bd_ring.len = len;
3053 	rx_ring->bd_ring.desc_size = desc_size;
3054 	rx_ring->bd_ring.addr = *rxch_addr;
3055 	rx_ring->bd_ring.wp = 0;
3056 	rx_ring->bd_ring.rp = 0;
3057 	rx_ring->buf_sz = buf_sz;
3058 	rx_ring->diliver_skb = NULL;
3059 	rx_ring->diliver_desc.ready = false;
3060 
3061 	for (i = 0; i < len; i++) {
3062 		skb = dev_alloc_skb(buf_sz);
3063 		if (!skb) {
3064 			ret = -ENOMEM;
3065 			goto err_free;
3066 		}
3067 
3068 		memset(skb->data, 0, buf_sz);
3069 		rx_ring->buf[i] = skb;
3070 		ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3071 					   buf_sz, i);
3072 		if (ret) {
3073 			rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3074 			dev_kfree_skb_any(skb);
3075 			rx_ring->buf[i] = NULL;
3076 			goto err_free;
3077 		}
3078 	}
3079 
3080 	return 0;
3081 
3082 err_free:
3083 	allocated = i;
3084 	for (i = 0; i < allocated; i++) {
3085 		skb = rx_ring->buf[i];
3086 		if (!skb)
3087 			continue;
3088 		dma = *((dma_addr_t *)skb->cb);
3089 		dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3090 		dev_kfree_skb(skb);
3091 		rx_ring->buf[i] = NULL;
3092 	}
3093 
3094 	head = rx_ring->bd_ring.head;
3095 	dma = rx_ring->bd_ring.dma;
3096 	dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3097 
3098 	rx_ring->bd_ring.head = NULL;
3099 err:
3100 	return ret;
3101 }
3102 
3103 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3104 				    struct pci_dev *pdev)
3105 {
3106 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3107 	struct rtw89_pci_rx_ring *rx_ring;
3108 	u32 desc_size;
3109 	u32 len;
3110 	int i, rx_allocated;
3111 	int ret;
3112 
3113 	for (i = 0; i < RTW89_RXCH_NUM; i++) {
3114 		rx_ring = &rtwpci->rx_rings[i];
3115 		desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3116 		len = RTW89_PCI_RXBD_NUM_MAX;
3117 		ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3118 					      desc_size, len, i);
3119 		if (ret) {
3120 			rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3121 			goto err_free;
3122 		}
3123 	}
3124 
3125 	return 0;
3126 
3127 err_free:
3128 	rx_allocated = i;
3129 	for (i = 0; i < rx_allocated; i++) {
3130 		rx_ring = &rtwpci->rx_rings[i];
3131 		rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3132 	}
3133 
3134 	return ret;
3135 }
3136 
3137 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3138 				     struct pci_dev *pdev)
3139 {
3140 	int ret;
3141 
3142 	ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3143 	if (ret) {
3144 		rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3145 		goto err;
3146 	}
3147 
3148 	ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3149 	if (ret) {
3150 		rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3151 		goto err_free_tx_rings;
3152 	}
3153 
3154 	return 0;
3155 
3156 err_free_tx_rings:
3157 	rtw89_pci_free_tx_rings(rtwdev, pdev);
3158 err:
3159 	return ret;
3160 }
3161 
3162 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3163 			       struct rtw89_pci *rtwpci)
3164 {
3165 	skb_queue_head_init(&rtwpci->h2c_queue);
3166 	skb_queue_head_init(&rtwpci->h2c_release_queue);
3167 }
3168 
3169 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3170 				    struct pci_dev *pdev)
3171 {
3172 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3173 	int ret;
3174 
3175 	ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3176 	if (ret) {
3177 		rtw89_err(rtwdev, "failed to setup pci mapping\n");
3178 		goto err;
3179 	}
3180 
3181 	ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3182 	if (ret) {
3183 		rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3184 		goto err_pci_unmap;
3185 	}
3186 
3187 	rtw89_pci_h2c_init(rtwdev, rtwpci);
3188 
3189 	spin_lock_init(&rtwpci->irq_lock);
3190 	spin_lock_init(&rtwpci->trx_lock);
3191 
3192 	return 0;
3193 
3194 err_pci_unmap:
3195 	rtw89_pci_clear_mapping(rtwdev, pdev);
3196 err:
3197 	return ret;
3198 }
3199 
3200 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3201 				     struct pci_dev *pdev)
3202 {
3203 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3204 
3205 	rtw89_pci_free_trx_rings(rtwdev, pdev);
3206 	rtw89_pci_clear_mapping(rtwdev, pdev);
3207 	rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3208 				skb_queue_len(&rtwpci->h2c_queue), true);
3209 }
3210 
3211 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3212 {
3213 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3214 	const struct rtw89_chip_info *chip = rtwdev->chip;
3215 	u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN;
3216 
3217 	if (chip->chip_id == RTL8851B)
3218 		hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND;
3219 
3220 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3221 
3222 	if (rtwpci->under_recovery) {
3223 		rtwpci->intrs[0] = hs0isr_ind_int_en;
3224 		rtwpci->intrs[1] = 0;
3225 	} else {
3226 		rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3227 				   B_AX_RXDMA_INT_EN |
3228 				   B_AX_RXP1DMA_INT_EN |
3229 				   B_AX_RPQDMA_INT_EN |
3230 				   B_AX_RXDMA_STUCK_INT_EN |
3231 				   B_AX_RDU_INT_EN |
3232 				   B_AX_RPQBD_FULL_INT_EN |
3233 				   hs0isr_ind_int_en;
3234 
3235 		rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3236 	}
3237 }
3238 EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3239 
3240 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3241 {
3242 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3243 
3244 	rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
3245 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3246 	rtwpci->intrs[0] = 0;
3247 	rtwpci->intrs[1] = 0;
3248 }
3249 
3250 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3251 {
3252 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3253 
3254 	rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
3255 			    B_AX_HS1ISR_IND_INT_EN |
3256 			    B_AX_HS0ISR_IND_INT_EN;
3257 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3258 	rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3259 			   B_AX_RXDMA_INT_EN |
3260 			   B_AX_RXP1DMA_INT_EN |
3261 			   B_AX_RPQDMA_INT_EN |
3262 			   B_AX_RXDMA_STUCK_INT_EN |
3263 			   B_AX_RDU_INT_EN |
3264 			   B_AX_RPQBD_FULL_INT_EN;
3265 	rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3266 }
3267 
3268 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3269 {
3270 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3271 
3272 	rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
3273 			    B_AX_HS0ISR_IND_INT_EN;
3274 	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3275 	rtwpci->intrs[0] = 0;
3276 	rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3277 }
3278 
3279 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3280 {
3281 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3282 
3283 	if (rtwpci->under_recovery)
3284 		rtw89_pci_recovery_intr_mask_v1(rtwdev);
3285 	else if (rtwpci->low_power)
3286 		rtw89_pci_low_power_intr_mask_v1(rtwdev);
3287 	else
3288 		rtw89_pci_default_intr_mask_v1(rtwdev);
3289 }
3290 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
3291 
3292 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3293 				 struct pci_dev *pdev)
3294 {
3295 	unsigned long flags = 0;
3296 	int ret;
3297 
3298 	flags |= PCI_IRQ_LEGACY | PCI_IRQ_MSI;
3299 	ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
3300 	if (ret < 0) {
3301 		rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3302 		goto err;
3303 	}
3304 
3305 	ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3306 					rtw89_pci_interrupt_handler,
3307 					rtw89_pci_interrupt_threadfn,
3308 					IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3309 	if (ret) {
3310 		rtw89_err(rtwdev, "failed to request threaded irq\n");
3311 		goto err_free_vector;
3312 	}
3313 
3314 	rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3315 
3316 	return 0;
3317 
3318 err_free_vector:
3319 	pci_free_irq_vectors(pdev);
3320 err:
3321 	return ret;
3322 }
3323 
3324 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3325 			       struct pci_dev *pdev)
3326 {
3327 	devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3328 	pci_free_irq_vectors(pdev);
3329 }
3330 
3331 static u16 gray_code_to_bin(u16 gray_code, u32 bit_num)
3332 {
3333 	u16 bin = 0, gray_bit;
3334 	u32 bit_idx;
3335 
3336 	for (bit_idx = 0; bit_idx < bit_num; bit_idx++) {
3337 		gray_bit = (gray_code >> bit_idx) & 0x1;
3338 		if (bit_num - bit_idx > 1)
3339 			gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1;
3340 		bin |= (gray_bit << bit_idx);
3341 	}
3342 
3343 	return bin;
3344 }
3345 
3346 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3347 {
3348 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3349 	struct pci_dev *pdev = rtwpci->pdev;
3350 	u16 val16, filter_out_val;
3351 	u32 val, phy_offset;
3352 	int ret;
3353 
3354 	if (rtwdev->chip->chip_id != RTL8852C)
3355 		return 0;
3356 
3357 	val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3358 	if (val == B_AX_ASPM_CTRL_L1)
3359 		return 0;
3360 
3361 	ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
3362 	if (ret)
3363 		return ret;
3364 
3365 	val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
3366 	if (val == RTW89_PCIE_GEN1_SPEED) {
3367 		phy_offset = R_RAC_DIRECT_OFFSET_G1;
3368 	} else if (val == RTW89_PCIE_GEN2_SPEED) {
3369 		phy_offset = R_RAC_DIRECT_OFFSET_G2;
3370 		val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3371 		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3372 				  val16 | B_PCIE_BIT_PINOUT_DIS);
3373 		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3374 				  val16 & ~B_PCIE_BIT_RD_SEL);
3375 
3376 		val16 = rtw89_read16_mask(rtwdev,
3377 					  phy_offset + RAC_ANA1F * RAC_MULT,
3378 					  FILTER_OUT_EQ_MASK);
3379 		val16 = gray_code_to_bin(val16, hweight16(val16));
3380 		filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3381 					      RAC_MULT);
3382 		filter_out_val &= ~REG_FILTER_OUT_MASK;
3383 		filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
3384 
3385 		rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3386 			      filter_out_val);
3387 		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3388 				  B_BAC_EQ_SEL);
3389 		rtw89_write16_set(rtwdev,
3390 				  R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
3391 				  B_PCIE_BIT_PSAVE);
3392 	} else {
3393 		return -EOPNOTSUPP;
3394 	}
3395 	rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3396 			  B_PCIE_BIT_PSAVE);
3397 
3398 	return 0;
3399 }
3400 
3401 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3402 {
3403 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3404 	int ret;
3405 
3406 	if (rtw89_pci_disable_clkreq)
3407 		return;
3408 
3409 	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3410 					  PCIE_CLKDLY_HW_30US);
3411 	if (ret)
3412 		rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3413 
3414 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3415 		if (enable)
3416 			ret = rtw89_pci_config_byte_set(rtwdev,
3417 							RTW89_PCIE_L1_CTRL,
3418 							RTW89_PCIE_BIT_CLK);
3419 		else
3420 			ret = rtw89_pci_config_byte_clr(rtwdev,
3421 							RTW89_PCIE_L1_CTRL,
3422 							RTW89_PCIE_BIT_CLK);
3423 		if (ret)
3424 			rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3425 				  enable ? "set" : "unset", ret);
3426 	} else if (chip_id == RTL8852C) {
3427 		rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3428 				  B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
3429 		if (enable)
3430 			rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3431 					  B_AX_CLK_REQ_N);
3432 		else
3433 			rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3434 					  B_AX_CLK_REQ_N);
3435 	}
3436 }
3437 
3438 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3439 {
3440 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3441 	u8 value = 0;
3442 	int ret;
3443 
3444 	if (rtw89_pci_disable_aspm_l1)
3445 		return;
3446 
3447 	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
3448 	if (ret)
3449 		rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3450 
3451 	value &= ~(RTW89_L1DLY_MASK | RTW89_L0DLY_MASK);
3452 	value |= FIELD_PREP(RTW89_L1DLY_MASK, PCIE_L1DLY_16US) |
3453 		 FIELD_PREP(RTW89_L0DLY_MASK, PCIE_L0SDLY_4US);
3454 
3455 	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
3456 	if (ret)
3457 		rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3458 
3459 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3460 		if (enable)
3461 			ret = rtw89_pci_config_byte_set(rtwdev,
3462 							RTW89_PCIE_L1_CTRL,
3463 							RTW89_PCIE_BIT_L1);
3464 		else
3465 			ret = rtw89_pci_config_byte_clr(rtwdev,
3466 							RTW89_PCIE_L1_CTRL,
3467 							RTW89_PCIE_BIT_L1);
3468 	} else if (chip_id == RTL8852C) {
3469 		if (enable)
3470 			rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3471 					  B_AX_ASPM_CTRL_L1);
3472 		else
3473 			rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3474 					  B_AX_ASPM_CTRL_L1);
3475 	}
3476 	if (ret)
3477 		rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3478 			  enable ? "set" : "unset", ret);
3479 }
3480 
3481 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3482 {
3483 	struct rtw89_traffic_stats *stats = &rtwdev->stats;
3484 	enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
3485 	enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
3486 	u32 val = 0;
3487 
3488 	if (!rtwdev->scanning &&
3489 	    (tx_tfc_lv >= RTW89_TFC_HIGH || rx_tfc_lv >= RTW89_TFC_HIGH))
3490 		val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
3491 		      FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
3492 		      FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
3493 		      FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
3494 
3495 	rtw89_write32(rtwdev, R_AX_INT_MIT_RX, val);
3496 }
3497 
3498 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
3499 {
3500 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3501 	struct pci_dev *pdev = rtwpci->pdev;
3502 	u16 link_ctrl;
3503 	int ret;
3504 
3505 	/* Though there is standard PCIE configuration space to set the
3506 	 * link control register, but by Realtek's design, driver should
3507 	 * check if host supports CLKREQ/ASPM to enable the HW module.
3508 	 *
3509 	 * These functions are implemented by two HW modules associated,
3510 	 * one is responsible to access PCIE configuration space to
3511 	 * follow the host settings, and another is in charge of doing
3512 	 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
3513 	 * the host does not support it, and due to some reasons or wrong
3514 	 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
3515 	 * loss if HW misbehaves on the link.
3516 	 *
3517 	 * Hence it's designed that driver should first check the PCIE
3518 	 * configuration space is sync'ed and enabled, then driver can turn
3519 	 * on the other module that is actually working on the mechanism.
3520 	 */
3521 	ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
3522 	if (ret) {
3523 		rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
3524 		return;
3525 	}
3526 
3527 	if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
3528 		rtw89_pci_clkreq_set(rtwdev, true);
3529 
3530 	if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
3531 		rtw89_pci_aspm_set(rtwdev, true);
3532 }
3533 
3534 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
3535 {
3536 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3537 	int ret;
3538 
3539 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3540 		if (enable)
3541 			ret = rtw89_pci_config_byte_set(rtwdev,
3542 							RTW89_PCIE_TIMER_CTRL,
3543 							RTW89_PCIE_BIT_L1SUB);
3544 		else
3545 			ret = rtw89_pci_config_byte_clr(rtwdev,
3546 							RTW89_PCIE_TIMER_CTRL,
3547 							RTW89_PCIE_BIT_L1SUB);
3548 		if (ret)
3549 			rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
3550 				  enable ? "set" : "unset", ret);
3551 	} else if (chip_id == RTL8852C) {
3552 		ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
3553 						RTW89_PCIE_BIT_ASPM_L11 |
3554 						RTW89_PCIE_BIT_PCI_L11);
3555 		if (ret)
3556 			rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
3557 		if (enable)
3558 			rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3559 					  B_AX_L1SUB_DISABLE);
3560 		else
3561 			rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3562 					  B_AX_L1SUB_DISABLE);
3563 	}
3564 }
3565 
3566 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
3567 {
3568 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3569 	struct pci_dev *pdev = rtwpci->pdev;
3570 	u32 l1ss_cap_ptr, l1ss_ctrl;
3571 
3572 	if (rtw89_pci_disable_l1ss)
3573 		return;
3574 
3575 	l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
3576 	if (!l1ss_cap_ptr)
3577 		return;
3578 
3579 	pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl);
3580 
3581 	if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
3582 		rtw89_pci_l1ss_set(rtwdev, true);
3583 }
3584 
3585 static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
3586 {
3587 	int ret = 0;
3588 	u32 sts;
3589 	u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
3590 
3591 	ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
3592 				       10, 1000, false, rtwdev,
3593 				       R_AX_PCIE_DMA_BUSY1);
3594 	if (ret) {
3595 		rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
3596 			  rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
3597 		return -EINVAL;
3598 	}
3599 	return ret;
3600 }
3601 
3602 static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
3603 {
3604 	u32 val;
3605 	int ret;
3606 
3607 	if (rtwdev->chip->chip_id == RTL8852C)
3608 		return 0;
3609 
3610 	rtw89_pci_ctrl_dma_all(rtwdev, false);
3611 	ret = rtw89_pci_poll_io_idle(rtwdev);
3612 	if (ret) {
3613 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3614 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
3615 			    "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
3616 			    R_AX_DBG_ERR_FLAG, val);
3617 		if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
3618 			rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
3619 		if (val & B_AX_RX_STUCK)
3620 			rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
3621 		rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3622 		ret = rtw89_pci_poll_io_idle(rtwdev);
3623 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3624 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
3625 			    "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
3626 			    R_AX_DBG_ERR_FLAG, val);
3627 	}
3628 
3629 	return ret;
3630 }
3631 
3632 
3633 
3634 static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev)
3635 {
3636 	int ret = 0;
3637 	u32 val32, sts;
3638 
3639 	val32 = B_AX_RST_BDRAM;
3640 	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
3641 
3642 	ret = read_poll_timeout_atomic(rtw89_read32, sts,
3643 				       (sts & B_AX_RST_BDRAM) == 0x0, 1, 100,
3644 				       true, rtwdev, R_AX_PCIE_INIT_CFG1);
3645 	return ret;
3646 }
3647 
3648 static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
3649 {
3650 	u32 ret;
3651 
3652 	if (rtwdev->chip->chip_id == RTL8852C)
3653 		return 0;
3654 
3655 	rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
3656 	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3657 	rtw89_pci_clr_idx_all(rtwdev);
3658 
3659 	ret = rtw89_pci_rst_bdram(rtwdev);
3660 	if (ret)
3661 		return ret;
3662 
3663 	rtw89_pci_ctrl_dma_all(rtwdev, true);
3664 	return ret;
3665 }
3666 
3667 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
3668 					  enum rtw89_lv1_rcvy_step step)
3669 {
3670 	int ret;
3671 
3672 	switch (step) {
3673 	case RTW89_LV1_RCVY_STEP_1:
3674 		ret = rtw89_pci_lv1rst_stop_dma(rtwdev);
3675 		if (ret)
3676 			rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
3677 
3678 		break;
3679 
3680 	case RTW89_LV1_RCVY_STEP_2:
3681 		ret = rtw89_pci_lv1rst_start_dma(rtwdev);
3682 		if (ret)
3683 			rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
3684 		break;
3685 
3686 	default:
3687 		return -EINVAL;
3688 	}
3689 
3690 	return ret;
3691 }
3692 
3693 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
3694 {
3695 	rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
3696 		   rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
3697 	rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
3698 		   rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
3699 	rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
3700 		   rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
3701 }
3702 
3703 static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
3704 {
3705 	struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
3706 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3707 	unsigned long flags;
3708 	int work_done;
3709 
3710 	rtwdev->napi_budget_countdown = budget;
3711 
3712 	rtw89_pci_clear_isr0(rtwdev, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT);
3713 	work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3714 	if (work_done == budget)
3715 		return budget;
3716 
3717 	rtw89_pci_clear_isr0(rtwdev, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | B_AX_RDU_INT);
3718 	work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3719 	if (work_done < budget && napi_complete_done(napi, work_done)) {
3720 		spin_lock_irqsave(&rtwpci->irq_lock, flags);
3721 		if (likely(rtwpci->running))
3722 			rtw89_chip_enable_intr(rtwdev, rtwpci);
3723 		spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
3724 	}
3725 
3726 	return work_done;
3727 }
3728 
3729 static int __maybe_unused rtw89_pci_suspend(struct device *dev)
3730 {
3731 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
3732 	struct rtw89_dev *rtwdev = hw->priv;
3733 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3734 
3735 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3736 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3737 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3738 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3739 		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
3740 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3741 		rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
3742 				  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3743 	} else {
3744 		rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3745 				  B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3746 	}
3747 
3748 	return 0;
3749 }
3750 
3751 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
3752 {
3753 	if (rtwdev->chip->chip_id == RTL8852C)
3754 		return;
3755 
3756 	/* Hardware need write the reg twice to ensure the setting work */
3757 	rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3758 				    RTW89_PCIE_BIT_CFG_RST_MSTATE);
3759 	rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3760 				    RTW89_PCIE_BIT_CFG_RST_MSTATE);
3761 }
3762 
3763 static int __maybe_unused rtw89_pci_resume(struct device *dev)
3764 {
3765 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
3766 	struct rtw89_dev *rtwdev = hw->priv;
3767 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3768 
3769 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3770 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3771 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3772 	if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3773 		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
3774 				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3775 		rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
3776 				  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3777 	} else {
3778 		rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3779 				  B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3780 		rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3781 				  B_AX_SEL_REQ_ENTR_L1);
3782 	}
3783 	rtw89_pci_l2_hci_ldo(rtwdev);
3784 	rtw89_pci_filter_out(rtwdev);
3785 	rtw89_pci_link_cfg(rtwdev);
3786 	rtw89_pci_l1ss_cfg(rtwdev);
3787 
3788 	return 0;
3789 }
3790 
3791 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
3792 EXPORT_SYMBOL(rtw89_pm_ops);
3793 
3794 static const struct rtw89_hci_ops rtw89_pci_ops = {
3795 	.tx_write	= rtw89_pci_ops_tx_write,
3796 	.tx_kick_off	= rtw89_pci_ops_tx_kick_off,
3797 	.flush_queues	= rtw89_pci_ops_flush_queues,
3798 	.reset		= rtw89_pci_ops_reset,
3799 	.start		= rtw89_pci_ops_start,
3800 	.stop		= rtw89_pci_ops_stop,
3801 	.pause		= rtw89_pci_ops_pause,
3802 	.switch_mode	= rtw89_pci_ops_switch_mode,
3803 	.recalc_int_mit = rtw89_pci_recalc_int_mit,
3804 
3805 	.read8		= rtw89_pci_ops_read8,
3806 	.read16		= rtw89_pci_ops_read16,
3807 	.read32		= rtw89_pci_ops_read32,
3808 	.write8		= rtw89_pci_ops_write8,
3809 	.write16	= rtw89_pci_ops_write16,
3810 	.write32	= rtw89_pci_ops_write32,
3811 
3812 	.mac_pre_init	= rtw89_pci_ops_mac_pre_init,
3813 	.mac_post_init	= rtw89_pci_ops_mac_post_init,
3814 	.deinit		= rtw89_pci_ops_deinit,
3815 
3816 	.check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
3817 	.mac_lv1_rcvy	= rtw89_pci_ops_mac_lv1_recovery,
3818 	.dump_err_status = rtw89_pci_ops_dump_err_status,
3819 	.napi_poll	= rtw89_pci_napi_poll,
3820 
3821 	.recovery_start = rtw89_pci_ops_recovery_start,
3822 	.recovery_complete = rtw89_pci_ops_recovery_complete,
3823 
3824 	.ctrl_txdma_ch	= rtw89_pci_ctrl_txdma_ch_pcie,
3825 	.ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_pcie,
3826 	.ctrl_trxhci	= rtw89_pci_ctrl_dma_trx,
3827 	.poll_txdma_ch	= rtw89_poll_txdma_ch_idle_pcie,
3828 	.clr_idx_all	= rtw89_pci_clr_idx_all,
3829 	.clear		= rtw89_pci_clear_resource,
3830 	.disable_intr	= rtw89_pci_disable_intr_lock,
3831 	.enable_intr	= rtw89_pci_enable_intr_lock,
3832 	.rst_bdram	= rtw89_pci_rst_bdram_pcie,
3833 };
3834 
3835 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3836 {
3837 	struct rtw89_dev *rtwdev;
3838 	const struct rtw89_driver_info *info;
3839 	const struct rtw89_pci_info *pci_info;
3840 	int ret;
3841 
3842 	info = (const struct rtw89_driver_info *)id->driver_data;
3843 
3844 	rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
3845 					  sizeof(struct rtw89_pci),
3846 					  info->chip);
3847 	if (!rtwdev) {
3848 		dev_err(&pdev->dev, "failed to allocate hw\n");
3849 		return -ENOMEM;
3850 	}
3851 
3852 	pci_info = info->bus.pci;
3853 
3854 	rtwdev->pci_info = info->bus.pci;
3855 	rtwdev->hci.ops = &rtw89_pci_ops;
3856 	rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
3857 	rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
3858 	rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
3859 
3860 	SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
3861 
3862 	ret = rtw89_core_init(rtwdev);
3863 	if (ret) {
3864 		rtw89_err(rtwdev, "failed to initialise core\n");
3865 		goto err_release_hw;
3866 	}
3867 
3868 	ret = rtw89_pci_claim_device(rtwdev, pdev);
3869 	if (ret) {
3870 		rtw89_err(rtwdev, "failed to claim pci device\n");
3871 		goto err_core_deinit;
3872 	}
3873 
3874 	ret = rtw89_pci_setup_resource(rtwdev, pdev);
3875 	if (ret) {
3876 		rtw89_err(rtwdev, "failed to setup pci resource\n");
3877 		goto err_declaim_pci;
3878 	}
3879 
3880 	ret = rtw89_chip_info_setup(rtwdev);
3881 	if (ret) {
3882 		rtw89_err(rtwdev, "failed to setup chip information\n");
3883 		goto err_clear_resource;
3884 	}
3885 
3886 	rtw89_pci_filter_out(rtwdev);
3887 	rtw89_pci_link_cfg(rtwdev);
3888 	rtw89_pci_l1ss_cfg(rtwdev);
3889 
3890 	rtw89_core_napi_init(rtwdev);
3891 
3892 	ret = rtw89_pci_request_irq(rtwdev, pdev);
3893 	if (ret) {
3894 		rtw89_err(rtwdev, "failed to request pci irq\n");
3895 		goto err_deinit_napi;
3896 	}
3897 
3898 	ret = rtw89_core_register(rtwdev);
3899 	if (ret) {
3900 		rtw89_err(rtwdev, "failed to register core\n");
3901 		goto err_free_irq;
3902 	}
3903 
3904 	return 0;
3905 
3906 err_free_irq:
3907 	rtw89_pci_free_irq(rtwdev, pdev);
3908 err_deinit_napi:
3909 	rtw89_core_napi_deinit(rtwdev);
3910 err_clear_resource:
3911 	rtw89_pci_clear_resource(rtwdev, pdev);
3912 err_declaim_pci:
3913 	rtw89_pci_declaim_device(rtwdev, pdev);
3914 err_core_deinit:
3915 	rtw89_core_deinit(rtwdev);
3916 err_release_hw:
3917 	rtw89_free_ieee80211_hw(rtwdev);
3918 
3919 	return ret;
3920 }
3921 EXPORT_SYMBOL(rtw89_pci_probe);
3922 
3923 void rtw89_pci_remove(struct pci_dev *pdev)
3924 {
3925 	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
3926 	struct rtw89_dev *rtwdev;
3927 
3928 	rtwdev = hw->priv;
3929 
3930 	rtw89_pci_free_irq(rtwdev, pdev);
3931 	rtw89_core_napi_deinit(rtwdev);
3932 	rtw89_core_unregister(rtwdev);
3933 	rtw89_pci_clear_resource(rtwdev, pdev);
3934 	rtw89_pci_declaim_device(rtwdev, pdev);
3935 	rtw89_core_deinit(rtwdev);
3936 	rtw89_free_ieee80211_hw(rtwdev);
3937 }
3938 EXPORT_SYMBOL(rtw89_pci_remove);
3939 
3940 MODULE_AUTHOR("Realtek Corporation");
3941 MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver");
3942 MODULE_LICENSE("Dual BSD/GPL");
3943