1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2020 Realtek Corporation
3 */
4
5 #include <linux/pci.h>
6
7 #include "mac.h"
8 #include "pci.h"
9 #include "reg.h"
10 #include "ser.h"
11
12 static bool rtw89_pci_disable_clkreq;
13 static bool rtw89_pci_disable_aspm_l1;
14 static bool rtw89_pci_disable_l1ss;
15 module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
16 module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
17 module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
18 MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
19 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
20 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
21
rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev * rtwdev,u32 * phy_offset)22 static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev,
23 u32 *phy_offset)
24 {
25 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
26 struct pci_dev *pdev = rtwpci->pdev;
27 u32 val;
28 int ret;
29
30 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
31 if (ret)
32 return ret;
33
34 val = u32_get_bits(val, RTW89_BCFG_LINK_SPEED_MASK);
35 if (val == RTW89_PCIE_GEN1_SPEED) {
36 *phy_offset = R_RAC_DIRECT_OFFSET_G1;
37 } else if (val == RTW89_PCIE_GEN2_SPEED) {
38 *phy_offset = R_RAC_DIRECT_OFFSET_G2;
39 } else {
40 rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val);
41 return -EFAULT;
42 }
43
44 return 0;
45 }
46
rtw89_pci_rst_bdram_ax(struct rtw89_dev * rtwdev)47 static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
48 {
49 u32 val;
50 int ret;
51
52 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
53
54 ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
55 1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
56 rtwdev, R_AX_PCIE_INIT_CFG1);
57
58 return ret;
59 }
60
rtw89_pci_dma_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring,u32 cur_idx,bool tx)61 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
62 struct rtw89_pci_dma_ring *bd_ring,
63 u32 cur_idx, bool tx)
64 {
65 const struct rtw89_pci_info *info = rtwdev->pci_info;
66 u32 cnt, cur_rp, wp, rp, len;
67
68 rp = bd_ring->rp;
69 wp = bd_ring->wp;
70 len = bd_ring->len;
71
72 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
73 if (tx) {
74 cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
75 } else {
76 if (info->rx_ring_eq_is_full)
77 wp += 1;
78
79 cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
80 }
81
82 bd_ring->rp = cur_rp;
83
84 return cnt;
85 }
86
rtw89_pci_txbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)87 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
88 struct rtw89_pci_tx_ring *tx_ring)
89 {
90 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
91 u32 addr_idx = bd_ring->addr.idx;
92 u32 cnt, idx;
93
94 idx = rtw89_read32(rtwdev, addr_idx);
95 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
96
97 return cnt;
98 }
99
rtw89_pci_release_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,u32 cnt,bool release_all)100 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
101 struct rtw89_pci *rtwpci,
102 u32 cnt, bool release_all)
103 {
104 struct rtw89_pci_tx_data *tx_data;
105 struct sk_buff *skb;
106 u32 qlen;
107
108 while (cnt--) {
109 skb = skb_dequeue(&rtwpci->h2c_queue);
110 if (!skb) {
111 rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
112 return;
113 }
114 skb_queue_tail(&rtwpci->h2c_release_queue, skb);
115 }
116
117 qlen = skb_queue_len(&rtwpci->h2c_release_queue);
118 if (!release_all)
119 qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
120
121 while (qlen--) {
122 skb = skb_dequeue(&rtwpci->h2c_release_queue);
123 if (!skb) {
124 rtw89_err(rtwdev, "failed to release fwcmd\n");
125 return;
126 }
127 tx_data = RTW89_PCI_TX_SKB_CB(skb);
128 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
129 DMA_TO_DEVICE);
130 dev_kfree_skb_any(skb);
131 }
132 }
133
rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)134 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
135 struct rtw89_pci *rtwpci)
136 {
137 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
138 u32 cnt;
139
140 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
141 if (!cnt)
142 return;
143 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
144 }
145
rtw89_pci_rxbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)146 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
147 struct rtw89_pci_rx_ring *rx_ring)
148 {
149 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
150 u32 addr_idx = bd_ring->addr.idx;
151 u32 cnt, idx;
152
153 idx = rtw89_read32(rtwdev, addr_idx);
154 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
155
156 return cnt;
157 }
158
rtw89_pci_sync_skb_for_cpu(struct rtw89_dev * rtwdev,struct sk_buff * skb)159 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
160 struct sk_buff *skb)
161 {
162 struct rtw89_pci_rx_info *rx_info;
163 dma_addr_t dma;
164
165 rx_info = RTW89_PCI_RX_SKB_CB(skb);
166 dma = rx_info->dma;
167 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
168 DMA_FROM_DEVICE);
169 }
170
rtw89_pci_sync_skb_for_device(struct rtw89_dev * rtwdev,struct sk_buff * skb)171 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
172 struct sk_buff *skb)
173 {
174 struct rtw89_pci_rx_info *rx_info;
175 dma_addr_t dma;
176
177 rx_info = RTW89_PCI_RX_SKB_CB(skb);
178 dma = rx_info->dma;
179 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
180 DMA_FROM_DEVICE);
181 }
182
rtw89_pci_rxbd_info_update(struct rtw89_dev * rtwdev,struct sk_buff * skb)183 static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
184 struct sk_buff *skb)
185 {
186 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
187 struct rtw89_pci_rxbd_info *rxbd_info;
188 __le32 info;
189
190 rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
191 info = rxbd_info->dword;
192
193 rx_info->fs = le32_get_bits(info, RTW89_PCI_RXBD_FS);
194 rx_info->ls = le32_get_bits(info, RTW89_PCI_RXBD_LS);
195 rx_info->len = le32_get_bits(info, RTW89_PCI_RXBD_WRITE_SIZE);
196 rx_info->tag = le32_get_bits(info, RTW89_PCI_RXBD_TAG);
197 }
198
rtw89_pci_validate_rx_tag(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb)199 static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev,
200 struct rtw89_pci_rx_ring *rx_ring,
201 struct sk_buff *skb)
202 {
203 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
204 const struct rtw89_pci_info *info = rtwdev->pci_info;
205 u32 target_rx_tag;
206
207 if (!info->check_rx_tag)
208 return 0;
209
210 /* valid range is 1 ~ 0x1FFF */
211 if (rx_ring->target_rx_tag == 0)
212 target_rx_tag = 1;
213 else
214 target_rx_tag = rx_ring->target_rx_tag;
215
216 if (rx_info->tag != target_rx_tag) {
217 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "mismatch RX tag 0x%x 0x%x\n",
218 rx_info->tag, target_rx_tag);
219 return -EAGAIN;
220 }
221
222 return 0;
223 }
224
225 static
rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb)226 int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev,
227 struct rtw89_pci_rx_ring *rx_ring,
228 struct sk_buff *skb)
229 {
230 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
231 int rx_tag_retry = 100;
232 int ret;
233
234 do {
235 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
236 rtw89_pci_rxbd_info_update(rtwdev, skb);
237
238 ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb);
239 if (ret != -EAGAIN)
240 break;
241 } while (rx_tag_retry--);
242
243 /* update target rx_tag for next RX */
244 rx_ring->target_rx_tag = rx_info->tag + 1;
245
246 return ret;
247 }
248
rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev * rtwdev,bool enable)249 static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable)
250 {
251 const struct rtw89_pci_info *info = rtwdev->pci_info;
252 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
253 const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
254
255 if (enable) {
256 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
257 if (dma_stop2->addr)
258 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
259 } else {
260 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
261 if (dma_stop2->addr)
262 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
263 }
264 }
265
rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev * rtwdev,bool enable)266 static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable)
267 {
268 const struct rtw89_pci_info *info = rtwdev->pci_info;
269 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
270
271 if (enable)
272 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
273 else
274 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
275 }
276
277 static bool
rtw89_skb_put_rx_data(struct rtw89_dev * rtwdev,bool fs,bool ls,struct sk_buff * new,const struct sk_buff * skb,u32 offset,const struct rtw89_pci_rx_info * rx_info,const struct rtw89_rx_desc_info * desc_info)278 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
279 struct sk_buff *new,
280 const struct sk_buff *skb, u32 offset,
281 const struct rtw89_pci_rx_info *rx_info,
282 const struct rtw89_rx_desc_info *desc_info)
283 {
284 u32 copy_len = rx_info->len - offset;
285
286 if (unlikely(skb_tailroom(new) < copy_len)) {
287 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
288 "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
289 rx_info->len, desc_info->pkt_size, offset, fs, ls);
290 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
291 skb->data, rx_info->len);
292 /* length of a single segment skb is desc_info->pkt_size */
293 if (fs && ls) {
294 copy_len = desc_info->pkt_size;
295 } else {
296 rtw89_info(rtwdev, "drop rx data due to invalid length\n");
297 return false;
298 }
299 }
300
301 skb_put_data(new, skb->data + offset, copy_len);
302
303 return true;
304 }
305
rtw89_pci_get_rx_skb_idx(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring)306 static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
307 struct rtw89_pci_dma_ring *bd_ring)
308 {
309 const struct rtw89_pci_info *info = rtwdev->pci_info;
310 u32 wp = bd_ring->wp;
311
312 if (!info->rx_ring_eq_is_full)
313 return wp;
314
315 if (++wp >= bd_ring->len)
316 wp = 0;
317
318 return wp;
319 }
320
rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)321 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
322 struct rtw89_pci_rx_ring *rx_ring)
323 {
324 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
325 struct rtw89_pci_rx_info *rx_info;
326 struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
327 struct sk_buff *new = rx_ring->diliver_skb;
328 struct sk_buff *skb;
329 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
330 u32 skb_idx;
331 u32 offset;
332 u32 cnt = 1;
333 bool fs, ls;
334 int ret;
335
336 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
337 skb = rx_ring->buf[skb_idx];
338
339 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
340 if (ret) {
341 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
342 bd_ring->wp, ret);
343 goto err_sync_device;
344 }
345
346 rx_info = RTW89_PCI_RX_SKB_CB(skb);
347 fs = rx_info->fs;
348 ls = rx_info->ls;
349
350 if (fs) {
351 if (new) {
352 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
353 "skb should not be ready before first segment start\n");
354 goto err_sync_device;
355 }
356 if (desc_info->ready) {
357 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
358 goto err_sync_device;
359 }
360
361 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
362
363 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
364 if (!new)
365 goto err_sync_device;
366
367 rx_ring->diliver_skb = new;
368
369 /* first segment has RX desc */
370 offset = desc_info->offset + desc_info->rxd_len;
371 } else {
372 offset = sizeof(struct rtw89_pci_rxbd_info);
373 if (!new) {
374 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
375 goto err_sync_device;
376 }
377 }
378 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
379 goto err_sync_device;
380 rtw89_pci_sync_skb_for_device(rtwdev, skb);
381 rtw89_pci_rxbd_increase(rx_ring, 1);
382
383 if (!desc_info->ready) {
384 rtw89_warn(rtwdev, "no rx desc information\n");
385 goto err_free_resource;
386 }
387 if (ls) {
388 rtw89_core_rx(rtwdev, desc_info, new);
389 rx_ring->diliver_skb = NULL;
390 desc_info->ready = false;
391 }
392
393 return cnt;
394
395 err_sync_device:
396 rtw89_pci_sync_skb_for_device(rtwdev, skb);
397 rtw89_pci_rxbd_increase(rx_ring, 1);
398 err_free_resource:
399 if (new)
400 dev_kfree_skb_any(new);
401 rx_ring->diliver_skb = NULL;
402 desc_info->ready = false;
403
404 return cnt;
405 }
406
rtw89_pci_rxbd_deliver(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)407 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
408 struct rtw89_pci_rx_ring *rx_ring,
409 u32 cnt)
410 {
411 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
412 u32 rx_cnt;
413
414 while (cnt && rtwdev->napi_budget_countdown > 0) {
415 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
416 if (!rx_cnt) {
417 rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
418
419 /* skip the rest RXBD bufs */
420 rtw89_pci_rxbd_increase(rx_ring, cnt);
421 break;
422 }
423
424 cnt -= rx_cnt;
425 }
426
427 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
428 }
429
rtw89_pci_poll_rxq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)430 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
431 struct rtw89_pci *rtwpci, int budget)
432 {
433 struct rtw89_pci_rx_ring *rx_ring;
434 int countdown = rtwdev->napi_budget_countdown;
435 u32 cnt;
436
437 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
438
439 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
440 if (!cnt)
441 return 0;
442
443 cnt = min_t(u32, budget, cnt);
444
445 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
446
447 /* In case of flushing pending SKBs, the countdown may exceed. */
448 if (rtwdev->napi_budget_countdown <= 0)
449 return budget;
450
451 return budget - countdown;
452 }
453
rtw89_pci_tx_status(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct sk_buff * skb,u8 tx_status)454 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
455 struct rtw89_pci_tx_ring *tx_ring,
456 struct sk_buff *skb, u8 tx_status)
457 {
458 struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
459 struct ieee80211_tx_info *info;
460
461 rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
462
463 info = IEEE80211_SKB_CB(skb);
464 ieee80211_tx_info_clear_status(info);
465
466 if (info->flags & IEEE80211_TX_CTL_NO_ACK)
467 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
468 if (tx_status == RTW89_TX_DONE) {
469 info->flags |= IEEE80211_TX_STAT_ACK;
470 tx_ring->tx_acked++;
471 } else {
472 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
473 rtw89_debug(rtwdev, RTW89_DBG_FW,
474 "failed to TX of status %x\n", tx_status);
475 switch (tx_status) {
476 case RTW89_TX_RETRY_LIMIT:
477 tx_ring->tx_retry_lmt++;
478 break;
479 case RTW89_TX_LIFE_TIME:
480 tx_ring->tx_life_time++;
481 break;
482 case RTW89_TX_MACID_DROP:
483 tx_ring->tx_mac_id_drop++;
484 break;
485 default:
486 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
487 break;
488 }
489 }
490
491 ieee80211_tx_status_ni(rtwdev->hw, skb);
492 }
493
rtw89_pci_reclaim_txbd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)494 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
495 {
496 struct rtw89_pci_tx_wd *txwd;
497 u32 cnt;
498
499 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
500 while (cnt--) {
501 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
502 if (!txwd) {
503 rtw89_warn(rtwdev, "No busy txwd pages available\n");
504 break;
505 }
506
507 list_del_init(&txwd->list);
508
509 /* this skb has been freed by RPP */
510 if (skb_queue_len(&txwd->queue) == 0)
511 rtw89_pci_enqueue_txwd(tx_ring, txwd);
512 }
513 }
514
rtw89_pci_release_busy_txwd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)515 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
516 struct rtw89_pci_tx_ring *tx_ring)
517 {
518 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
519 struct rtw89_pci_tx_wd *txwd;
520 int i;
521
522 for (i = 0; i < wd_ring->page_num; i++) {
523 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
524 if (!txwd)
525 break;
526
527 list_del_init(&txwd->list);
528 }
529 }
530
rtw89_pci_release_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,u16 seq,u8 tx_status)531 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
532 struct rtw89_pci_tx_ring *tx_ring,
533 struct rtw89_pci_tx_wd *txwd, u16 seq,
534 u8 tx_status)
535 {
536 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
537 struct rtw89_pci_tx_data *tx_data;
538 struct sk_buff *skb, *tmp;
539 u8 txch = tx_ring->txch;
540
541 if (!list_empty(&txwd->list)) {
542 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
543 /* In low power mode, RPP can receive before updating of TX BD.
544 * In normal mode, it should not happen so give it a warning.
545 */
546 if (!rtwpci->low_power && !list_empty(&txwd->list))
547 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
548 txch, seq);
549 }
550
551 skb_queue_walk_safe(&txwd->queue, skb, tmp) {
552 skb_unlink(skb, &txwd->queue);
553
554 tx_data = RTW89_PCI_TX_SKB_CB(skb);
555 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
556 DMA_TO_DEVICE);
557
558 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
559 }
560
561 if (list_empty(&txwd->list))
562 rtw89_pci_enqueue_txwd(tx_ring, txwd);
563 }
564
rtw89_pci_release_rpp(struct rtw89_dev * rtwdev,struct rtw89_pci_rpp_fmt * rpp)565 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
566 struct rtw89_pci_rpp_fmt *rpp)
567 {
568 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
569 struct rtw89_pci_tx_ring *tx_ring;
570 struct rtw89_pci_tx_wd_ring *wd_ring;
571 struct rtw89_pci_tx_wd *txwd;
572 u16 seq;
573 u8 qsel, tx_status, txch;
574
575 seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ);
576 qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL);
577 tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS);
578 txch = rtw89_core_get_ch_dma(rtwdev, qsel);
579
580 if (txch == RTW89_TXCH_CH12) {
581 rtw89_warn(rtwdev, "should no fwcmd release report\n");
582 return;
583 }
584
585 tx_ring = &rtwpci->tx_rings[txch];
586 wd_ring = &tx_ring->wd_ring;
587 txwd = &wd_ring->pages[seq];
588
589 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
590 }
591
rtw89_pci_release_pending_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)592 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
593 struct rtw89_pci_tx_ring *tx_ring)
594 {
595 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
596 struct rtw89_pci_tx_wd *txwd;
597 int i;
598
599 for (i = 0; i < wd_ring->page_num; i++) {
600 txwd = &wd_ring->pages[i];
601
602 if (!list_empty(&txwd->list))
603 continue;
604
605 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
606 }
607 }
608
rtw89_pci_release_tx_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 max_cnt)609 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
610 struct rtw89_pci_rx_ring *rx_ring,
611 u32 max_cnt)
612 {
613 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
614 struct rtw89_pci_rx_info *rx_info;
615 struct rtw89_pci_rpp_fmt *rpp;
616 struct rtw89_rx_desc_info desc_info = {};
617 struct sk_buff *skb;
618 u32 cnt = 0;
619 u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
620 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
621 u32 skb_idx;
622 u32 offset;
623 int ret;
624
625 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
626 skb = rx_ring->buf[skb_idx];
627
628 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
629 if (ret) {
630 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
631 bd_ring->wp, ret);
632 goto err_sync_device;
633 }
634
635 rx_info = RTW89_PCI_RX_SKB_CB(skb);
636 if (!rx_info->fs || !rx_info->ls) {
637 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
638 return cnt;
639 }
640
641 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
642
643 /* first segment has RX desc */
644 offset = desc_info.offset + desc_info.rxd_len;
645 for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
646 rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset);
647 rtw89_pci_release_rpp(rtwdev, rpp);
648 }
649
650 rtw89_pci_sync_skb_for_device(rtwdev, skb);
651 rtw89_pci_rxbd_increase(rx_ring, 1);
652 cnt++;
653
654 return cnt;
655
656 err_sync_device:
657 rtw89_pci_sync_skb_for_device(rtwdev, skb);
658 return 0;
659 }
660
rtw89_pci_release_tx(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)661 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
662 struct rtw89_pci_rx_ring *rx_ring,
663 u32 cnt)
664 {
665 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
666 u32 release_cnt;
667
668 while (cnt) {
669 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
670 if (!release_cnt) {
671 rtw89_err(rtwdev, "failed to release TX skbs\n");
672
673 /* skip the rest RXBD bufs */
674 rtw89_pci_rxbd_increase(rx_ring, cnt);
675 break;
676 }
677
678 cnt -= release_cnt;
679 }
680
681 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
682 }
683
rtw89_pci_poll_rpq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)684 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
685 struct rtw89_pci *rtwpci, int budget)
686 {
687 struct rtw89_pci_rx_ring *rx_ring;
688 u32 cnt;
689 int work_done;
690
691 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
692
693 spin_lock_bh(&rtwpci->trx_lock);
694
695 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
696 if (cnt == 0)
697 goto out_unlock;
698
699 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
700
701 out_unlock:
702 spin_unlock_bh(&rtwpci->trx_lock);
703
704 /* always release all RPQ */
705 work_done = min_t(int, cnt, budget);
706 rtwdev->napi_budget_countdown -= work_done;
707
708 return work_done;
709 }
710
rtw89_pci_isr_rxd_unavail(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)711 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
712 struct rtw89_pci *rtwpci)
713 {
714 struct rtw89_pci_rx_ring *rx_ring;
715 struct rtw89_pci_dma_ring *bd_ring;
716 u32 reg_idx;
717 u16 hw_idx, hw_idx_next, host_idx;
718 int i;
719
720 for (i = 0; i < RTW89_RXCH_NUM; i++) {
721 rx_ring = &rtwpci->rx_rings[i];
722 bd_ring = &rx_ring->bd_ring;
723
724 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
725 hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
726 host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
727 hw_idx_next = (hw_idx + 1) % bd_ring->len;
728
729 if (hw_idx_next == host_idx)
730 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
731
732 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
733 "%d RXD unavailable, idx=0x%08x, len=%d\n",
734 i, reg_idx, bd_ring->len);
735 }
736 }
737
rtw89_pci_recognize_intrs(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)738 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
739 struct rtw89_pci *rtwpci,
740 struct rtw89_pci_isrs *isrs)
741 {
742 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
743 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
744 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
745
746 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
747 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
748 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
749 }
750 EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
751
rtw89_pci_recognize_intrs_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)752 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
753 struct rtw89_pci *rtwpci,
754 struct rtw89_pci_isrs *isrs)
755 {
756 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
757 isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
758 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
759 isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
760 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
761 isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
762 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
763
764 if (isrs->halt_c2h_isrs)
765 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
766 if (isrs->isrs[0])
767 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
768 if (isrs->isrs[1])
769 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
770 }
771 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
772
rtw89_pci_recognize_intrs_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)773 void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
774 struct rtw89_pci *rtwpci,
775 struct rtw89_pci_isrs *isrs)
776 {
777 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
778 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ?
779 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
780 isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ?
781 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
782 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1];
783
784 if (isrs->halt_c2h_isrs)
785 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
786 if (isrs->isrs[0])
787 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]);
788 if (isrs->isrs[1])
789 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
790 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
791 }
792 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2);
793
rtw89_pci_enable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)794 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
795 {
796 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
797 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
798 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
799 }
800 EXPORT_SYMBOL(rtw89_pci_enable_intr);
801
rtw89_pci_disable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)802 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
803 {
804 rtw89_write32(rtwdev, R_AX_HIMR0, 0);
805 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
806 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
807 }
808 EXPORT_SYMBOL(rtw89_pci_disable_intr);
809
rtw89_pci_enable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)810 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
811 {
812 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
813 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
814 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
815 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
816 }
817 EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
818
rtw89_pci_disable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)819 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
820 {
821 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
822 }
823 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
824
rtw89_pci_enable_intr_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)825 void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
826 {
827 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
828 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]);
829 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
830 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
831 }
832 EXPORT_SYMBOL(rtw89_pci_enable_intr_v2);
833
rtw89_pci_disable_intr_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)834 void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
835 {
836 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
837 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
838 }
839 EXPORT_SYMBOL(rtw89_pci_disable_intr_v2);
840
rtw89_pci_ops_recovery_start(struct rtw89_dev * rtwdev)841 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
842 {
843 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
844 unsigned long flags;
845
846 spin_lock_irqsave(&rtwpci->irq_lock, flags);
847 rtw89_chip_disable_intr(rtwdev, rtwpci);
848 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
849 rtw89_chip_enable_intr(rtwdev, rtwpci);
850 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
851 }
852
rtw89_pci_ops_recovery_complete(struct rtw89_dev * rtwdev)853 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
854 {
855 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
856 unsigned long flags;
857
858 spin_lock_irqsave(&rtwpci->irq_lock, flags);
859 rtw89_chip_disable_intr(rtwdev, rtwpci);
860 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
861 rtw89_chip_enable_intr(rtwdev, rtwpci);
862 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
863 }
864
rtw89_pci_low_power_interrupt_handler(struct rtw89_dev * rtwdev)865 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
866 {
867 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
868 int budget = NAPI_POLL_WEIGHT;
869
870 /* To prevent RXQ get stuck due to run out of budget. */
871 rtwdev->napi_budget_countdown = budget;
872
873 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
874 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
875 }
876
rtw89_pci_interrupt_threadfn(int irq,void * dev)877 static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
878 {
879 struct rtw89_dev *rtwdev = dev;
880 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
881 const struct rtw89_pci_info *info = rtwdev->pci_info;
882 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
883 struct rtw89_pci_isrs isrs;
884 unsigned long flags;
885
886 spin_lock_irqsave(&rtwpci->irq_lock, flags);
887 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
888 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
889
890 if (unlikely(isrs.isrs[0] & gen_def->isr_rdu))
891 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
892
893 if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_halt_c2h))
894 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
895
896 if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_wdt_timeout))
897 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
898
899 if (unlikely(rtwpci->under_recovery))
900 goto enable_intr;
901
902 if (unlikely(rtwpci->low_power)) {
903 rtw89_pci_low_power_interrupt_handler(rtwdev);
904 goto enable_intr;
905 }
906
907 if (likely(rtwpci->running)) {
908 local_bh_disable();
909 napi_schedule(&rtwdev->napi);
910 local_bh_enable();
911 }
912
913 return IRQ_HANDLED;
914
915 enable_intr:
916 spin_lock_irqsave(&rtwpci->irq_lock, flags);
917 if (likely(rtwpci->running))
918 rtw89_chip_enable_intr(rtwdev, rtwpci);
919 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
920 return IRQ_HANDLED;
921 }
922
rtw89_pci_interrupt_handler(int irq,void * dev)923 static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
924 {
925 struct rtw89_dev *rtwdev = dev;
926 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
927 unsigned long flags;
928 irqreturn_t irqret = IRQ_WAKE_THREAD;
929
930 spin_lock_irqsave(&rtwpci->irq_lock, flags);
931
932 /* If interrupt event is on the road, it is still trigger interrupt
933 * even we have done pci_stop() to turn off IMR.
934 */
935 if (unlikely(!rtwpci->running)) {
936 irqret = IRQ_HANDLED;
937 goto exit;
938 }
939
940 rtw89_chip_disable_intr(rtwdev, rtwpci);
941 exit:
942 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
943
944 return irqret;
945 }
946
947 #define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \
948 [RTW89_TXCH_##ch_idx] = { \
949 .num = R_##gen##_##txch##_TXBD_NUM ##v, \
950 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \
951 .bdram = 0, \
952 .desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \
953 .desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \
954 }
955
956 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
957 [RTW89_TXCH_##txch] = { \
958 .num = R_AX_##txch##_TXBD_NUM ##v, \
959 .idx = R_AX_##txch##_TXBD_IDX ##v, \
960 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
961 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
962 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
963 }
964
965 #define DEF_TXCHADDRS(info, txch, v...) \
966 [RTW89_TXCH_##txch] = { \
967 .num = R_AX_##txch##_TXBD_NUM, \
968 .idx = R_AX_##txch##_TXBD_IDX, \
969 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
970 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
971 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
972 }
973
974 #define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \
975 [RTW89_RXCH_##ch_idx] = { \
976 .num = R_##gen##_##rxch##_RXBD_NUM ##v, \
977 .idx = R_##gen##_##rxch##_RXBD_IDX ##v, \
978 .desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \
979 .desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \
980 }
981
982 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
983 .tx = {
984 DEF_TXCHADDRS(info, ACH0),
985 DEF_TXCHADDRS(info, ACH1),
986 DEF_TXCHADDRS(info, ACH2),
987 DEF_TXCHADDRS(info, ACH3),
988 DEF_TXCHADDRS(info, ACH4),
989 DEF_TXCHADDRS(info, ACH5),
990 DEF_TXCHADDRS(info, ACH6),
991 DEF_TXCHADDRS(info, ACH7),
992 DEF_TXCHADDRS(info, CH8),
993 DEF_TXCHADDRS(info, CH9),
994 DEF_TXCHADDRS_TYPE1(info, CH10),
995 DEF_TXCHADDRS_TYPE1(info, CH11),
996 DEF_TXCHADDRS(info, CH12),
997 },
998 .rx = {
999 DEF_RXCHADDRS(AX, RXQ, RXQ),
1000 DEF_RXCHADDRS(AX, RPQ, RPQ),
1001 },
1002 };
1003 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
1004
1005 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
1006 .tx = {
1007 DEF_TXCHADDRS(info, ACH0, _V1),
1008 DEF_TXCHADDRS(info, ACH1, _V1),
1009 DEF_TXCHADDRS(info, ACH2, _V1),
1010 DEF_TXCHADDRS(info, ACH3, _V1),
1011 DEF_TXCHADDRS(info, ACH4, _V1),
1012 DEF_TXCHADDRS(info, ACH5, _V1),
1013 DEF_TXCHADDRS(info, ACH6, _V1),
1014 DEF_TXCHADDRS(info, ACH7, _V1),
1015 DEF_TXCHADDRS(info, CH8, _V1),
1016 DEF_TXCHADDRS(info, CH9, _V1),
1017 DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
1018 DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
1019 DEF_TXCHADDRS(info, CH12, _V1),
1020 },
1021 .rx = {
1022 DEF_RXCHADDRS(AX, RXQ, RXQ, _V1),
1023 DEF_RXCHADDRS(AX, RPQ, RPQ, _V1),
1024 },
1025 };
1026 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
1027
1028 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = {
1029 .tx = {
1030 DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1),
1031 DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1),
1032 DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1),
1033 DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1),
1034 DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1),
1035 DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1),
1036 DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1),
1037 DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1),
1038 DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1),
1039 DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1),
1040 DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1),
1041 DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1),
1042 DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1),
1043 },
1044 .rx = {
1045 DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1),
1046 DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1),
1047 },
1048 };
1049 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be);
1050
1051 #undef DEF_TXCHADDRS_TYPE1
1052 #undef DEF_TXCHADDRS
1053 #undef DEF_RXCHADDRS
1054
rtw89_pci_get_txch_addrs(struct rtw89_dev * rtwdev,enum rtw89_tx_channel txch,const struct rtw89_pci_ch_dma_addr ** addr)1055 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
1056 enum rtw89_tx_channel txch,
1057 const struct rtw89_pci_ch_dma_addr **addr)
1058 {
1059 const struct rtw89_pci_info *info = rtwdev->pci_info;
1060
1061 if (txch >= RTW89_TXCH_NUM)
1062 return -EINVAL;
1063
1064 *addr = &info->dma_addr_set->tx[txch];
1065
1066 return 0;
1067 }
1068
rtw89_pci_get_rxch_addrs(struct rtw89_dev * rtwdev,enum rtw89_rx_channel rxch,const struct rtw89_pci_ch_dma_addr ** addr)1069 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
1070 enum rtw89_rx_channel rxch,
1071 const struct rtw89_pci_ch_dma_addr **addr)
1072 {
1073 const struct rtw89_pci_info *info = rtwdev->pci_info;
1074
1075 if (rxch >= RTW89_RXCH_NUM)
1076 return -EINVAL;
1077
1078 *addr = &info->dma_addr_set->rx[rxch];
1079
1080 return 0;
1081 }
1082
rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring * ring)1083 static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
1084 {
1085 struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
1086
1087 /* reserved 1 desc check ring is full or not */
1088 if (bd_ring->rp > bd_ring->wp)
1089 return bd_ring->rp - bd_ring->wp - 1;
1090
1091 return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
1092 }
1093
1094 static
__rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev * rtwdev)1095 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
1096 {
1097 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1098 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
1099 u32 cnt;
1100
1101 spin_lock_bh(&rtwpci->trx_lock);
1102 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
1103 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1104 spin_unlock_bh(&rtwpci->trx_lock);
1105
1106 return cnt;
1107 }
1108
1109 static
__rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev * rtwdev,u8 txch)1110 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
1111 u8 txch)
1112 {
1113 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1114 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1115 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1116 u32 cnt;
1117
1118 spin_lock_bh(&rtwpci->trx_lock);
1119 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1120 if (txch != RTW89_TXCH_CH12)
1121 cnt = min(cnt, wd_ring->curr_num);
1122 spin_unlock_bh(&rtwpci->trx_lock);
1123
1124 return cnt;
1125 }
1126
__rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)1127 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1128 u8 txch)
1129 {
1130 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1131 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1132 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1133 const struct rtw89_chip_info *chip = rtwdev->chip;
1134 u32 bd_cnt, wd_cnt, min_cnt = 0;
1135 struct rtw89_pci_rx_ring *rx_ring;
1136 enum rtw89_debug_mask debug_mask;
1137 u32 cnt;
1138
1139 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
1140
1141 spin_lock_bh(&rtwpci->trx_lock);
1142 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1143 wd_cnt = wd_ring->curr_num;
1144
1145 if (wd_cnt == 0 || bd_cnt == 0) {
1146 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
1147 if (cnt)
1148 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
1149 else if (wd_cnt == 0)
1150 goto out_unlock;
1151
1152 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1153 if (bd_cnt == 0)
1154 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
1155 }
1156
1157 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1158 wd_cnt = wd_ring->curr_num;
1159 min_cnt = min(bd_cnt, wd_cnt);
1160 if (min_cnt == 0) {
1161 /* This message can be frequently shown in low power mode or
1162 * high traffic with small FIFO chips, and we have recognized it as normal
1163 * behavior, so print with mask RTW89_DBG_TXRX in these situations.
1164 */
1165 if (rtwpci->low_power || chip->small_fifo_size)
1166 debug_mask = RTW89_DBG_TXRX;
1167 else
1168 debug_mask = RTW89_DBG_UNEXP;
1169
1170 rtw89_debug(rtwdev, debug_mask,
1171 "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
1172 wd_cnt, bd_cnt);
1173 }
1174
1175 out_unlock:
1176 spin_unlock_bh(&rtwpci->trx_lock);
1177
1178 return min_cnt;
1179 }
1180
rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)1181 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1182 u8 txch)
1183 {
1184 if (rtwdev->hci.paused)
1185 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1186
1187 if (txch == RTW89_TXCH_CH12)
1188 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1189
1190 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1191 }
1192
__rtw89_pci_tx_kick_off(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1193 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1194 {
1195 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1196 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1197 u32 host_idx, addr;
1198
1199 spin_lock_bh(&rtwpci->trx_lock);
1200
1201 addr = bd_ring->addr.idx;
1202 host_idx = bd_ring->wp;
1203 rtw89_write16(rtwdev, addr, host_idx);
1204
1205 spin_unlock_bh(&rtwpci->trx_lock);
1206 }
1207
rtw89_pci_tx_bd_ring_update(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,int n_txbd)1208 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1209 int n_txbd)
1210 {
1211 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1212 u32 host_idx, len;
1213
1214 len = bd_ring->len;
1215 host_idx = bd_ring->wp + n_txbd;
1216 host_idx = host_idx < len ? host_idx : host_idx - len;
1217
1218 bd_ring->wp = host_idx;
1219 }
1220
rtw89_pci_ops_tx_kick_off(struct rtw89_dev * rtwdev,u8 txch)1221 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1222 {
1223 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1224 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1225
1226 if (rtwdev->hci.paused) {
1227 set_bit(txch, rtwpci->kick_map);
1228 return;
1229 }
1230
1231 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1232 }
1233
rtw89_pci_tx_kick_off_pending(struct rtw89_dev * rtwdev)1234 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1235 {
1236 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1237 struct rtw89_pci_tx_ring *tx_ring;
1238 int txch;
1239
1240 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1241 if (!test_and_clear_bit(txch, rtwpci->kick_map))
1242 continue;
1243
1244 tx_ring = &rtwpci->tx_rings[txch];
1245 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1246 }
1247 }
1248
__pci_flush_txch(struct rtw89_dev * rtwdev,u8 txch,bool drop)1249 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1250 {
1251 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1252 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1253 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1254 u32 cur_idx, cur_rp;
1255 u8 i;
1256
1257 /* Because the time taked by the I/O is a bit dynamic, it's hard to
1258 * define a reasonable fixed total timeout to use read_poll_timeout*
1259 * helper. Instead, we can ensure a reasonable polling times, so we
1260 * just use for loop with udelay here.
1261 */
1262 for (i = 0; i < 60; i++) {
1263 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1264 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1265 if (cur_rp == bd_ring->wp)
1266 return;
1267
1268 udelay(1);
1269 }
1270
1271 if (!drop)
1272 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1273 }
1274
__rtw89_pci_ops_flush_txchs(struct rtw89_dev * rtwdev,u32 txchs,bool drop)1275 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1276 bool drop)
1277 {
1278 const struct rtw89_pci_info *info = rtwdev->pci_info;
1279 u8 i;
1280
1281 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1282 /* It may be unnecessary to flush FWCMD queue. */
1283 if (i == RTW89_TXCH_CH12)
1284 continue;
1285 if (info->tx_dma_ch_mask & BIT(i))
1286 continue;
1287
1288 if (txchs & BIT(i))
1289 __pci_flush_txch(rtwdev, i, drop);
1290 }
1291 }
1292
rtw89_pci_ops_flush_queues(struct rtw89_dev * rtwdev,u32 queues,bool drop)1293 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1294 bool drop)
1295 {
1296 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1297 }
1298
rtw89_pci_fill_txaddr_info(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1299 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1300 void *txaddr_info_addr, u32 total_len,
1301 dma_addr_t dma, u8 *add_info_nr)
1302 {
1303 struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1304 __le16 option;
1305
1306 txaddr_info->length = cpu_to_le16(total_len);
1307 option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS | RTW89_PCI_ADDR_NUM(1));
1308 option |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_ADDR_HIGH_MASK);
1309 txaddr_info->option = option;
1310 txaddr_info->dma = cpu_to_le32(dma);
1311
1312 *add_info_nr = 1;
1313
1314 return sizeof(*txaddr_info);
1315 }
1316 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1317
rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1318 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1319 void *txaddr_info_addr, u32 total_len,
1320 dma_addr_t dma, u8 *add_info_nr)
1321 {
1322 struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1323 u32 remain = total_len;
1324 u32 len;
1325 u16 length_option;
1326 int n;
1327
1328 for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1329 len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1330 TXADDR_INFO_LENTHG_V1_MAX : remain;
1331 remain -= len;
1332
1333 length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1334 FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1335 FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1336 length_option |= u16_encode_bits(upper_32_bits(dma),
1337 B_PCIADDR_HIGH_SEL_V1_MASK);
1338 txaddr_info->length_opt = cpu_to_le16(length_option);
1339 txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1340 txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1341
1342 dma += len;
1343 txaddr_info++;
1344 }
1345
1346 WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1347 remain, total_len);
1348
1349 *add_info_nr = n;
1350
1351 return n * sizeof(*txaddr_info);
1352 }
1353 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1354
rtw89_pci_txwd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,struct rtw89_core_tx_request * tx_req)1355 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1356 struct rtw89_pci_tx_ring *tx_ring,
1357 struct rtw89_pci_tx_wd *txwd,
1358 struct rtw89_core_tx_request *tx_req)
1359 {
1360 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1361 const struct rtw89_chip_info *chip = rtwdev->chip;
1362 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1363 struct rtw89_pci_tx_wp_info *txwp_info;
1364 void *txaddr_info_addr;
1365 struct pci_dev *pdev = rtwpci->pdev;
1366 struct sk_buff *skb = tx_req->skb;
1367 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1368 struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
1369 bool en_wd_info = desc_info->en_wd_info;
1370 u32 txwd_len;
1371 u32 txwp_len;
1372 u32 txaddr_info_len;
1373 dma_addr_t dma;
1374 int ret;
1375
1376 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1377 if (dma_mapping_error(&pdev->dev, dma)) {
1378 rtw89_err(rtwdev, "failed to map skb dma data\n");
1379 ret = -EBUSY;
1380 goto err;
1381 }
1382
1383 tx_data->dma = dma;
1384 rcu_assign_pointer(skb_data->wait, NULL);
1385
1386 txwp_len = sizeof(*txwp_info);
1387 txwd_len = chip->txwd_body_size;
1388 txwd_len += en_wd_info ? chip->txwd_info_size : 0;
1389
1390 txwp_info = txwd->vaddr + txwd_len;
1391 txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1392 txwp_info->seq1 = 0;
1393 txwp_info->seq2 = 0;
1394 txwp_info->seq3 = 0;
1395
1396 tx_ring->tx_cnt++;
1397 txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1398 txaddr_info_len =
1399 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1400 dma, &desc_info->addr_info_nr);
1401
1402 txwd->len = txwd_len + txwp_len + txaddr_info_len;
1403
1404 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1405
1406 skb_queue_tail(&txwd->queue, skb);
1407
1408 return 0;
1409
1410 err:
1411 return ret;
1412 }
1413
rtw89_pci_fwcmd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1414 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1415 struct rtw89_pci_tx_ring *tx_ring,
1416 struct rtw89_pci_tx_bd_32 *txbd,
1417 struct rtw89_core_tx_request *tx_req)
1418 {
1419 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1420 const struct rtw89_chip_info *chip = rtwdev->chip;
1421 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1422 void *txdesc;
1423 int txdesc_size = chip->h2c_desc_size;
1424 struct pci_dev *pdev = rtwpci->pdev;
1425 struct sk_buff *skb = tx_req->skb;
1426 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1427 dma_addr_t dma;
1428 __le16 opt;
1429
1430 txdesc = skb_push(skb, txdesc_size);
1431 memset(txdesc, 0, txdesc_size);
1432 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1433
1434 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1435 if (dma_mapping_error(&pdev->dev, dma)) {
1436 rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1437 return -EBUSY;
1438 }
1439
1440 tx_data->dma = dma;
1441 opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS);
1442 opt |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_TXBD_OPT_DMA_HI);
1443 txbd->opt = opt;
1444 txbd->length = cpu_to_le16(skb->len);
1445 txbd->dma = cpu_to_le32(tx_data->dma);
1446 skb_queue_tail(&rtwpci->h2c_queue, skb);
1447
1448 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1449
1450 return 0;
1451 }
1452
rtw89_pci_txbd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1453 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1454 struct rtw89_pci_tx_ring *tx_ring,
1455 struct rtw89_pci_tx_bd_32 *txbd,
1456 struct rtw89_core_tx_request *tx_req)
1457 {
1458 struct rtw89_pci_tx_wd *txwd;
1459 __le16 opt;
1460 int ret;
1461
1462 /* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1463 * buffer with WD BODY only. So here we don't need to check the free
1464 * pages of the wd ring.
1465 */
1466 if (tx_ring->txch == RTW89_TXCH_CH12)
1467 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1468
1469 txwd = rtw89_pci_dequeue_txwd(tx_ring);
1470 if (!txwd) {
1471 rtw89_err(rtwdev, "no available TXWD\n");
1472 ret = -ENOSPC;
1473 goto err;
1474 }
1475
1476 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1477 if (ret) {
1478 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1479 goto err_enqueue_wd;
1480 }
1481
1482 list_add_tail(&txwd->list, &tx_ring->busy_pages);
1483
1484 opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS);
1485 opt |= le16_encode_bits(upper_32_bits(txwd->paddr), RTW89_PCI_TXBD_OPT_DMA_HI);
1486 txbd->opt = opt;
1487 txbd->length = cpu_to_le16(txwd->len);
1488 txbd->dma = cpu_to_le32(txwd->paddr);
1489
1490 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1491
1492 return 0;
1493
1494 err_enqueue_wd:
1495 rtw89_pci_enqueue_txwd(tx_ring, txwd);
1496 err:
1497 return ret;
1498 }
1499
rtw89_pci_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req,u8 txch)1500 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1501 u8 txch)
1502 {
1503 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1504 struct rtw89_pci_tx_ring *tx_ring;
1505 struct rtw89_pci_tx_bd_32 *txbd;
1506 u32 n_avail_txbd;
1507 int ret = 0;
1508
1509 /* check the tx type and dma channel for fw cmd queue */
1510 if ((txch == RTW89_TXCH_CH12 ||
1511 tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1512 (txch != RTW89_TXCH_CH12 ||
1513 tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1514 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1515 return -EINVAL;
1516 }
1517
1518 tx_ring = &rtwpci->tx_rings[txch];
1519 spin_lock_bh(&rtwpci->trx_lock);
1520
1521 n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring);
1522 if (n_avail_txbd == 0) {
1523 rtw89_err(rtwdev, "no available TXBD\n");
1524 ret = -ENOSPC;
1525 goto err_unlock;
1526 }
1527
1528 txbd = rtw89_pci_get_next_txbd(tx_ring);
1529 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1530 if (ret) {
1531 rtw89_err(rtwdev, "failed to submit TXBD\n");
1532 goto err_unlock;
1533 }
1534
1535 spin_unlock_bh(&rtwpci->trx_lock);
1536 return 0;
1537
1538 err_unlock:
1539 spin_unlock_bh(&rtwpci->trx_lock);
1540 return ret;
1541 }
1542
rtw89_pci_ops_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req)1543 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1544 {
1545 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1546 int ret;
1547
1548 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1549 if (ret) {
1550 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1551 return ret;
1552 }
1553
1554 return 0;
1555 }
1556
1557 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = {
1558 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1559 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1560 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1561 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1562 [RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1563 [RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1564 [RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1565 [RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1566 [RTW89_TXCH_CH8] = {.start_idx = 40, .max_num = 5, .min_num = 1},
1567 [RTW89_TXCH_CH9] = {.start_idx = 45, .max_num = 5, .min_num = 1},
1568 [RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1569 [RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1570 [RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1571 };
1572 EXPORT_SYMBOL(rtw89_bd_ram_table_dual);
1573
1574 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = {
1575 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1576 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1577 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1578 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1579 [RTW89_TXCH_CH8] = {.start_idx = 20, .max_num = 4, .min_num = 1},
1580 [RTW89_TXCH_CH9] = {.start_idx = 24, .max_num = 4, .min_num = 1},
1581 [RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1},
1582 };
1583 EXPORT_SYMBOL(rtw89_bd_ram_table_single);
1584
rtw89_pci_init_wp_16sel(struct rtw89_dev * rtwdev)1585 static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev)
1586 {
1587 const struct rtw89_pci_info *info = rtwdev->pci_info;
1588 u32 addr = info->wp_sel_addr;
1589 u32 val;
1590 int i;
1591
1592 if (!info->wp_sel_addr)
1593 return;
1594
1595 for (i = 0; i < 16; i += 4) {
1596 val = u32_encode_bits(i + 0, MASKBYTE0) |
1597 u32_encode_bits(i + 1, MASKBYTE1) |
1598 u32_encode_bits(i + 2, MASKBYTE2) |
1599 u32_encode_bits(i + 3, MASKBYTE3);
1600 rtw89_write32(rtwdev, addr + i, val);
1601 }
1602 }
1603
rtw89_pci_reset_trx_rings(struct rtw89_dev * rtwdev)1604 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1605 {
1606 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1607 const struct rtw89_pci_info *info = rtwdev->pci_info;
1608 const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table;
1609 struct rtw89_pci_tx_ring *tx_ring;
1610 struct rtw89_pci_rx_ring *rx_ring;
1611 struct rtw89_pci_dma_ring *bd_ring;
1612 const struct rtw89_pci_bd_ram *bd_ram;
1613 u32 addr_num;
1614 u32 addr_idx;
1615 u32 addr_bdram;
1616 u32 addr_desa_l;
1617 u32 val32;
1618 int i;
1619
1620 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1621 if (info->tx_dma_ch_mask & BIT(i))
1622 continue;
1623
1624 tx_ring = &rtwpci->tx_rings[i];
1625 bd_ring = &tx_ring->bd_ring;
1626 bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL;
1627 addr_num = bd_ring->addr.num;
1628 addr_bdram = bd_ring->addr.bdram;
1629 addr_desa_l = bd_ring->addr.desa_l;
1630 bd_ring->wp = 0;
1631 bd_ring->rp = 0;
1632
1633 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1634 if (addr_bdram && bd_ram) {
1635 val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1636 FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1637 FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1638
1639 rtw89_write32(rtwdev, addr_bdram, val32);
1640 }
1641 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1642 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1643 }
1644
1645 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1646 rx_ring = &rtwpci->rx_rings[i];
1647 bd_ring = &rx_ring->bd_ring;
1648 addr_num = bd_ring->addr.num;
1649 addr_idx = bd_ring->addr.idx;
1650 addr_desa_l = bd_ring->addr.desa_l;
1651 if (info->rx_ring_eq_is_full)
1652 bd_ring->wp = bd_ring->len - 1;
1653 else
1654 bd_ring->wp = 0;
1655 bd_ring->rp = 0;
1656 rx_ring->diliver_skb = NULL;
1657 rx_ring->diliver_desc.ready = false;
1658 rx_ring->target_rx_tag = 0;
1659
1660 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1661 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1662 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1663
1664 if (info->rx_ring_eq_is_full)
1665 rtw89_write16(rtwdev, addr_idx, bd_ring->wp);
1666 }
1667
1668 rtw89_pci_init_wp_16sel(rtwdev);
1669 }
1670
rtw89_pci_release_tx_ring(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1671 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1672 struct rtw89_pci_tx_ring *tx_ring)
1673 {
1674 rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1675 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1676 }
1677
rtw89_pci_ops_reset(struct rtw89_dev * rtwdev)1678 void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1679 {
1680 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1681 const struct rtw89_pci_info *info = rtwdev->pci_info;
1682 int txch;
1683
1684 rtw89_pci_reset_trx_rings(rtwdev);
1685
1686 spin_lock_bh(&rtwpci->trx_lock);
1687 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1688 if (info->tx_dma_ch_mask & BIT(txch))
1689 continue;
1690 if (txch == RTW89_TXCH_CH12) {
1691 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1692 skb_queue_len(&rtwpci->h2c_queue), true);
1693 continue;
1694 }
1695 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1696 }
1697 spin_unlock_bh(&rtwpci->trx_lock);
1698 }
1699
rtw89_pci_enable_intr_lock(struct rtw89_dev * rtwdev)1700 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1701 {
1702 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1703 unsigned long flags;
1704
1705 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1706 rtwpci->running = true;
1707 rtw89_chip_enable_intr(rtwdev, rtwpci);
1708 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1709 }
1710
rtw89_pci_disable_intr_lock(struct rtw89_dev * rtwdev)1711 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1712 {
1713 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1714 unsigned long flags;
1715
1716 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1717 rtwpci->running = false;
1718 rtw89_chip_disable_intr(rtwdev, rtwpci);
1719 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1720 }
1721
rtw89_pci_ops_start(struct rtw89_dev * rtwdev)1722 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1723 {
1724 rtw89_core_napi_start(rtwdev);
1725 rtw89_pci_enable_intr_lock(rtwdev);
1726
1727 return 0;
1728 }
1729
rtw89_pci_ops_stop(struct rtw89_dev * rtwdev)1730 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1731 {
1732 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1733 struct pci_dev *pdev = rtwpci->pdev;
1734
1735 rtw89_pci_disable_intr_lock(rtwdev);
1736 synchronize_irq(pdev->irq);
1737 rtw89_core_napi_stop(rtwdev);
1738 }
1739
rtw89_pci_ops_pause(struct rtw89_dev * rtwdev,bool pause)1740 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1741 {
1742 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1743 struct pci_dev *pdev = rtwpci->pdev;
1744
1745 if (pause) {
1746 rtw89_pci_disable_intr_lock(rtwdev);
1747 synchronize_irq(pdev->irq);
1748 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1749 napi_synchronize(&rtwdev->napi);
1750 } else {
1751 rtw89_pci_enable_intr_lock(rtwdev);
1752 rtw89_pci_tx_kick_off_pending(rtwdev);
1753 }
1754 }
1755
1756 static
rtw89_pci_switch_bd_idx_addr(struct rtw89_dev * rtwdev,bool low_power)1757 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1758 {
1759 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1760 const struct rtw89_pci_info *info = rtwdev->pci_info;
1761 const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1762 const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1763 struct rtw89_pci_tx_ring *tx_ring;
1764 struct rtw89_pci_rx_ring *rx_ring;
1765 int i;
1766
1767 if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1768 return;
1769
1770 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1771 tx_ring = &rtwpci->tx_rings[i];
1772 tx_ring->bd_ring.addr.idx = low_power ?
1773 bd_idx_addr->tx_bd_addrs[i] :
1774 dma_addr_set->tx[i].idx;
1775 }
1776
1777 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1778 rx_ring = &rtwpci->rx_rings[i];
1779 rx_ring->bd_ring.addr.idx = low_power ?
1780 bd_idx_addr->rx_bd_addrs[i] :
1781 dma_addr_set->rx[i].idx;
1782 }
1783 }
1784
rtw89_pci_ops_switch_mode(struct rtw89_dev * rtwdev,bool low_power)1785 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1786 {
1787 enum rtw89_pci_intr_mask_cfg cfg;
1788
1789 WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1790
1791 cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
1792 rtw89_chip_config_intr_mask(rtwdev, cfg);
1793 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1794 }
1795
1796 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1797
rtw89_pci_ops_read32_cmac(struct rtw89_dev * rtwdev,u32 addr)1798 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1799 {
1800 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1801 u32 val = readl(rtwpci->mmap + addr);
1802 int count;
1803
1804 for (count = 0; ; count++) {
1805 if (val != RTW89_R32_DEAD)
1806 return val;
1807 if (count >= MAC_REG_POOL_COUNT) {
1808 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1809 return RTW89_R32_DEAD;
1810 }
1811 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1812 val = readl(rtwpci->mmap + addr);
1813 }
1814
1815 return val;
1816 }
1817
rtw89_pci_ops_read8(struct rtw89_dev * rtwdev,u32 addr)1818 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1819 {
1820 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1821 u32 addr32, val32, shift;
1822
1823 if (!ACCESS_CMAC(addr))
1824 return readb(rtwpci->mmap + addr);
1825
1826 addr32 = addr & ~0x3;
1827 shift = (addr & 0x3) * 8;
1828 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1829 return val32 >> shift;
1830 }
1831
rtw89_pci_ops_read16(struct rtw89_dev * rtwdev,u32 addr)1832 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1833 {
1834 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1835 u32 addr32, val32, shift;
1836
1837 if (!ACCESS_CMAC(addr))
1838 return readw(rtwpci->mmap + addr);
1839
1840 addr32 = addr & ~0x3;
1841 shift = (addr & 0x3) * 8;
1842 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1843 return val32 >> shift;
1844 }
1845
rtw89_pci_ops_read32(struct rtw89_dev * rtwdev,u32 addr)1846 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1847 {
1848 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1849
1850 if (!ACCESS_CMAC(addr))
1851 return readl(rtwpci->mmap + addr);
1852
1853 return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1854 }
1855
rtw89_pci_ops_write8(struct rtw89_dev * rtwdev,u32 addr,u8 data)1856 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1857 {
1858 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1859
1860 writeb(data, rtwpci->mmap + addr);
1861 }
1862
rtw89_pci_ops_write16(struct rtw89_dev * rtwdev,u32 addr,u16 data)1863 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1864 {
1865 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1866
1867 writew(data, rtwpci->mmap + addr);
1868 }
1869
rtw89_pci_ops_write32(struct rtw89_dev * rtwdev,u32 addr,u32 data)1870 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1871 {
1872 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1873
1874 writel(data, rtwpci->mmap + addr);
1875 }
1876
rtw89_pci_ctrl_dma_trx(struct rtw89_dev * rtwdev,bool enable)1877 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1878 {
1879 const struct rtw89_pci_info *info = rtwdev->pci_info;
1880
1881 if (enable)
1882 rtw89_write32_set(rtwdev, info->init_cfg_reg,
1883 info->rxhci_en_bit | info->txhci_en_bit);
1884 else
1885 rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1886 info->rxhci_en_bit | info->txhci_en_bit);
1887 }
1888
rtw89_pci_ctrl_dma_io(struct rtw89_dev * rtwdev,bool enable)1889 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1890 {
1891 const struct rtw89_pci_info *info = rtwdev->pci_info;
1892 const struct rtw89_reg_def *reg = &info->dma_io_stop;
1893
1894 if (enable)
1895 rtw89_write32_clr(rtwdev, reg->addr, reg->mask);
1896 else
1897 rtw89_write32_set(rtwdev, reg->addr, reg->mask);
1898 }
1899
rtw89_pci_ctrl_dma_all(struct rtw89_dev * rtwdev,bool enable)1900 void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1901 {
1902 rtw89_pci_ctrl_dma_io(rtwdev, enable);
1903 rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1904 }
1905
rtw89_pci_check_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 rw_bit)1906 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1907 {
1908 u16 val;
1909
1910 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1911
1912 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1913 switch (speed) {
1914 case PCIE_PHY_GEN1:
1915 if (addr < 0x20)
1916 val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
1917 else
1918 val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
1919 break;
1920 case PCIE_PHY_GEN2:
1921 if (addr < 0x20)
1922 val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
1923 else
1924 val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
1925 break;
1926 default:
1927 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
1928 return -EINVAL;
1929 }
1930 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
1931 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
1932
1933 return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
1934 false, rtwdev, R_AX_MDIO_CFG);
1935 }
1936
1937 static int
rtw89_read16_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 * val)1938 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
1939 {
1940 int ret;
1941
1942 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
1943 if (ret) {
1944 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
1945 return ret;
1946 }
1947 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
1948
1949 return 0;
1950 }
1951
1952 static int
rtw89_write16_mdio(struct rtw89_dev * rtwdev,u8 addr,u16 data,u8 speed)1953 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
1954 {
1955 int ret;
1956
1957 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
1958 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
1959 if (ret) {
1960 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
1961 return ret;
1962 }
1963
1964 return 0;
1965 }
1966
1967 static int
rtw89_write16_mdio_mask(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u16 data,u8 speed)1968 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
1969 {
1970 u32 shift;
1971 int ret;
1972 u16 val;
1973
1974 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1975 if (ret)
1976 return ret;
1977
1978 shift = __ffs(mask);
1979 val &= ~mask;
1980 val |= ((data << shift) & mask);
1981
1982 ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
1983 if (ret)
1984 return ret;
1985
1986 return 0;
1987 }
1988
rtw89_write16_mdio_set(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)1989 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1990 {
1991 int ret;
1992 u16 val;
1993
1994 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1995 if (ret)
1996 return ret;
1997 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
1998 if (ret)
1999 return ret;
2000
2001 return 0;
2002 }
2003
rtw89_write16_mdio_clr(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)2004 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2005 {
2006 int ret;
2007 u16 val;
2008
2009 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2010 if (ret)
2011 return ret;
2012 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
2013 if (ret)
2014 return ret;
2015
2016 return 0;
2017 }
2018
rtw89_dbi_write8(struct rtw89_dev * rtwdev,u16 addr,u8 data)2019 static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data)
2020 {
2021 u16 addr_2lsb = addr & B_AX_DBI_2LSB;
2022 u16 write_addr;
2023 u8 flag;
2024 int ret;
2025
2026 write_addr = addr & B_AX_DBI_ADDR_MSK;
2027 write_addr |= u16_encode_bits(BIT(addr_2lsb), B_AX_DBI_WREN_MSK);
2028 rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data);
2029 rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr);
2030 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16);
2031
2032 ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10,
2033 10 * RTW89_PCI_WR_RETRY_CNT, false,
2034 rtwdev, R_AX_DBI_FLAG + 2);
2035 if (ret)
2036 rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n",
2037 addr);
2038
2039 return ret;
2040 }
2041
rtw89_dbi_read8(struct rtw89_dev * rtwdev,u16 addr,u8 * value)2042 static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value)
2043 {
2044 u16 read_addr = addr & B_AX_DBI_ADDR_MSK;
2045 u8 flag;
2046 int ret;
2047
2048 rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr);
2049 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16);
2050
2051 ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10,
2052 10 * RTW89_PCI_WR_RETRY_CNT, false,
2053 rtwdev, R_AX_DBI_FLAG + 2);
2054 if (ret) {
2055 rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n",
2056 addr);
2057 return ret;
2058 }
2059
2060 read_addr = R_AX_DBI_RDATA + (addr & 3);
2061 *value = rtw89_read8(rtwdev, read_addr);
2062
2063 return 0;
2064 }
2065
rtw89_pci_write_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 data)2066 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2067 u8 data)
2068 {
2069 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2070 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2071 struct pci_dev *pdev = rtwpci->pdev;
2072 int ret;
2073
2074 ret = pci_write_config_byte(pdev, addr, data);
2075 if (!ret)
2076 return 0;
2077
2078 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2079 ret = rtw89_dbi_write8(rtwdev, addr, data);
2080
2081 return ret;
2082 }
2083
rtw89_pci_read_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 * value)2084 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2085 u8 *value)
2086 {
2087 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2088 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2089 struct pci_dev *pdev = rtwpci->pdev;
2090 int ret;
2091
2092 ret = pci_read_config_byte(pdev, addr, value);
2093 if (!ret)
2094 return 0;
2095
2096 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2097 ret = rtw89_dbi_read8(rtwdev, addr, value);
2098
2099 return ret;
2100 }
2101
rtw89_pci_config_byte_set(struct rtw89_dev * rtwdev,u16 addr,u8 bit)2102 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
2103 u8 bit)
2104 {
2105 u8 value;
2106 int ret;
2107
2108 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2109 if (ret)
2110 return ret;
2111
2112 value |= bit;
2113 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2114
2115 return ret;
2116 }
2117
rtw89_pci_config_byte_clr(struct rtw89_dev * rtwdev,u16 addr,u8 bit)2118 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
2119 u8 bit)
2120 {
2121 u8 value;
2122 int ret;
2123
2124 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2125 if (ret)
2126 return ret;
2127
2128 value &= ~bit;
2129 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2130
2131 return ret;
2132 }
2133
2134 static int
__get_target(struct rtw89_dev * rtwdev,u16 * target,enum rtw89_pcie_phy phy_rate)2135 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
2136 {
2137 u16 val, tar;
2138 int ret;
2139
2140 /* Enable counter */
2141 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
2142 if (ret)
2143 return ret;
2144 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2145 phy_rate);
2146 if (ret)
2147 return ret;
2148 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
2149 phy_rate);
2150 if (ret)
2151 return ret;
2152
2153 fsleep(300);
2154
2155 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
2156 if (ret)
2157 return ret;
2158 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2159 phy_rate);
2160 if (ret)
2161 return ret;
2162
2163 tar = tar & 0x0FFF;
2164 if (tar == 0 || tar == 0x0FFF) {
2165 rtw89_err(rtwdev, "[ERR]Get target failed.\n");
2166 return -EINVAL;
2167 }
2168
2169 *target = tar;
2170
2171 return 0;
2172 }
2173
rtw89_pci_autok_x(struct rtw89_dev * rtwdev)2174 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
2175 {
2176 int ret;
2177
2178 if (!rtw89_is_rtl885xb(rtwdev))
2179 return 0;
2180
2181 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
2182 PCIE_AUTOK_4, PCIE_PHY_GEN1);
2183 return ret;
2184 }
2185
rtw89_pci_auto_refclk_cal(struct rtw89_dev * rtwdev,bool autook_en)2186 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
2187 {
2188 enum rtw89_pcie_phy phy_rate;
2189 u16 val16, mgn_set, div_set, tar;
2190 u8 val8, bdr_ori;
2191 bool l1_flag = false;
2192 int ret = 0;
2193
2194 if (!rtw89_is_rtl885xb(rtwdev))
2195 return 0;
2196
2197 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
2198 if (ret) {
2199 rtw89_err(rtwdev, "[ERR]pci config read %X\n",
2200 RTW89_PCIE_PHY_RATE);
2201 return ret;
2202 }
2203
2204 if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
2205 phy_rate = PCIE_PHY_GEN1;
2206 } else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
2207 phy_rate = PCIE_PHY_GEN2;
2208 } else {
2209 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
2210 return -EOPNOTSUPP;
2211 }
2212 /* Disable L1BD */
2213 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
2214 if (ret) {
2215 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
2216 return ret;
2217 }
2218
2219 if (bdr_ori & RTW89_PCIE_BIT_L1) {
2220 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2221 bdr_ori & ~RTW89_PCIE_BIT_L1);
2222 if (ret) {
2223 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2224 RTW89_PCIE_L1_CTRL);
2225 return ret;
2226 }
2227 l1_flag = true;
2228 }
2229
2230 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2231 if (ret) {
2232 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2233 goto end;
2234 }
2235
2236 if (val16 & B_AX_CALIB_EN) {
2237 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
2238 val16 & ~B_AX_CALIB_EN, phy_rate);
2239 if (ret) {
2240 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2241 goto end;
2242 }
2243 }
2244
2245 if (!autook_en)
2246 goto end;
2247 /* Set div */
2248 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
2249 if (ret) {
2250 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2251 goto end;
2252 }
2253
2254 /* Obtain div and margin */
2255 ret = __get_target(rtwdev, &tar, phy_rate);
2256 if (ret) {
2257 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2258 goto end;
2259 }
2260
2261 mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
2262
2263 if (mgn_set >= 128) {
2264 div_set = 0x0003;
2265 mgn_set = 0x000F;
2266 } else if (mgn_set >= 64) {
2267 div_set = 0x0003;
2268 mgn_set >>= 3;
2269 } else if (mgn_set >= 32) {
2270 div_set = 0x0002;
2271 mgn_set >>= 2;
2272 } else if (mgn_set >= 16) {
2273 div_set = 0x0001;
2274 mgn_set >>= 1;
2275 } else if (mgn_set == 0) {
2276 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2277 goto end;
2278 } else {
2279 div_set = 0x0000;
2280 }
2281
2282 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2283 if (ret) {
2284 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2285 goto end;
2286 }
2287
2288 val16 |= u16_encode_bits(div_set, B_AX_DIV);
2289
2290 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2291 if (ret) {
2292 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2293 goto end;
2294 }
2295
2296 ret = __get_target(rtwdev, &tar, phy_rate);
2297 if (ret) {
2298 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2299 goto end;
2300 }
2301
2302 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2303 tar, div_set, mgn_set);
2304 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2305 (tar & 0x0FFF) | (mgn_set << 12), phy_rate);
2306 if (ret) {
2307 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2308 goto end;
2309 }
2310
2311 /* Enable function */
2312 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2313 if (ret) {
2314 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2315 goto end;
2316 }
2317
2318 /* CLK delay = 0 */
2319 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2320 PCIE_CLKDLY_HW_0);
2321
2322 end:
2323 /* Set L1BD to ori */
2324 if (l1_flag) {
2325 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2326 bdr_ori);
2327 if (ret) {
2328 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2329 RTW89_PCIE_L1_CTRL);
2330 return ret;
2331 }
2332 }
2333
2334 return ret;
2335 }
2336
rtw89_pci_deglitch_setting(struct rtw89_dev * rtwdev)2337 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2338 {
2339 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2340 int ret;
2341
2342 if (chip_id == RTL8852A) {
2343 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2344 PCIE_PHY_GEN1);
2345 if (ret)
2346 return ret;
2347 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2348 PCIE_PHY_GEN2);
2349 if (ret)
2350 return ret;
2351 } else if (chip_id == RTL8852C) {
2352 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2353 B_AX_DEGLITCH);
2354 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2355 B_AX_DEGLITCH);
2356 }
2357
2358 return 0;
2359 }
2360
rtw89_pci_disable_eq(struct rtw89_dev * rtwdev)2361 static void rtw89_pci_disable_eq(struct rtw89_dev *rtwdev)
2362 {
2363 u16 g1_oobs, g2_oobs;
2364 u32 backup_aspm;
2365 u32 phy_offset;
2366 u16 oobs_val;
2367 int ret;
2368
2369 if (rtwdev->chip->chip_id != RTL8852C)
2370 return;
2371
2372 g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2373 RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL);
2374 g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 +
2375 RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL);
2376 if (g1_oobs && g2_oobs)
2377 return;
2378
2379 backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1);
2380 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
2381
2382 ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, &phy_offset);
2383 if (ret)
2384 goto out;
2385
2386 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN);
2387 rtw89_write16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL);
2388 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL);
2389
2390 oobs_val = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT,
2391 OOBS_LEVEL_MASK);
2392
2393 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT,
2394 OOBS_SEN_MASK, oobs_val);
2395 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT,
2396 BAC_OOBS_SEL);
2397
2398 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT,
2399 OOBS_SEN_MASK, oobs_val);
2400 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT,
2401 BAC_OOBS_SEL);
2402
2403 out:
2404 rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, backup_aspm);
2405 }
2406
rtw89_pci_ber(struct rtw89_dev * rtwdev)2407 static void rtw89_pci_ber(struct rtw89_dev *rtwdev)
2408 {
2409 u32 phy_offset;
2410
2411 if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks))
2412 return;
2413
2414 phy_offset = R_RAC_DIRECT_OFFSET_G1;
2415 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL);
2416 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2417
2418 phy_offset = R_RAC_DIRECT_OFFSET_G2;
2419 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL);
2420 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2421 }
2422
rtw89_pci_rxdma_prefth(struct rtw89_dev * rtwdev)2423 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2424 {
2425 if (rtwdev->chip->chip_id != RTL8852A)
2426 return;
2427
2428 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2429 }
2430
rtw89_pci_l1off_pwroff(struct rtw89_dev * rtwdev)2431 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2432 {
2433 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2434
2435 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2436 return;
2437
2438 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2439 }
2440
rtw89_pci_l2_rxen_lat(struct rtw89_dev * rtwdev)2441 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2442 {
2443 int ret;
2444
2445 if (rtwdev->chip->chip_id != RTL8852A)
2446 return 0;
2447
2448 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2449 PCIE_PHY_GEN1);
2450 if (ret)
2451 return ret;
2452
2453 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2454 PCIE_PHY_GEN2);
2455 if (ret)
2456 return ret;
2457
2458 return 0;
2459 }
2460
rtw89_pci_aphy_pwrcut(struct rtw89_dev * rtwdev)2461 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2462 {
2463 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2464
2465 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2466 return;
2467
2468 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2469 }
2470
rtw89_pci_hci_ldo(struct rtw89_dev * rtwdev)2471 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2472 {
2473 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2474
2475 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2476 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2477 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2478 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2479 B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2480 } else if (rtwdev->chip->chip_id == RTL8852C) {
2481 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2482 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2483 }
2484 }
2485
rtw89_pci_dphy_delay(struct rtw89_dev * rtwdev)2486 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2487 {
2488 if (!rtw89_is_rtl885xb(rtwdev))
2489 return 0;
2490
2491 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2492 PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1);
2493 }
2494
rtw89_pci_power_wake(struct rtw89_dev * rtwdev,bool pwr_up)2495 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)
2496 {
2497 if (pwr_up)
2498 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2499 else
2500 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2501 }
2502
rtw89_pci_autoload_hang(struct rtw89_dev * rtwdev)2503 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2504 {
2505 if (rtwdev->chip->chip_id != RTL8852C)
2506 return;
2507
2508 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2509 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2510 }
2511
rtw89_pci_l12_vmain(struct rtw89_dev * rtwdev)2512 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2513 {
2514 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2515 return;
2516
2517 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2518 }
2519
rtw89_pci_gen2_force_ib(struct rtw89_dev * rtwdev)2520 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2521 {
2522 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2523 return;
2524
2525 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2526 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2527 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2528 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2529 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2530 }
2531
rtw89_pci_l1_ent_lat(struct rtw89_dev * rtwdev)2532 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2533 {
2534 if (rtwdev->chip->chip_id != RTL8852C)
2535 return;
2536
2537 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2538 }
2539
rtw89_pci_wd_exit_l1(struct rtw89_dev * rtwdev)2540 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2541 {
2542 if (rtwdev->chip->chip_id != RTL8852C)
2543 return;
2544
2545 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2546 }
2547
rtw89_pci_set_sic(struct rtw89_dev * rtwdev)2548 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2549 {
2550 if (rtwdev->chip->chip_id == RTL8852C)
2551 return;
2552
2553 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2554 B_AX_SIC_EN_FORCE_CLKREQ);
2555 }
2556
rtw89_pci_set_lbc(struct rtw89_dev * rtwdev)2557 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2558 {
2559 const struct rtw89_pci_info *info = rtwdev->pci_info;
2560 u32 lbc;
2561
2562 if (rtwdev->chip->chip_id == RTL8852C)
2563 return;
2564
2565 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2566 if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2567 lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER);
2568 lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2569 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2570 } else {
2571 lbc &= ~B_AX_LBC_EN;
2572 }
2573 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2574 }
2575
rtw89_pci_set_io_rcy(struct rtw89_dev * rtwdev)2576 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2577 {
2578 const struct rtw89_pci_info *info = rtwdev->pci_info;
2579 u32 val32;
2580
2581 if (rtwdev->chip->chip_id != RTL8852C)
2582 return;
2583
2584 if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2585 val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2586 info->io_rcy_tmr);
2587 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2588 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2589 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2590
2591 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2592 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2593 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2594 } else {
2595 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2596 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2597 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2598 }
2599
2600 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2601 }
2602
rtw89_pci_set_dbg(struct rtw89_dev * rtwdev)2603 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2604 {
2605 if (rtwdev->chip->chip_id == RTL8852C)
2606 return;
2607
2608 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2609 B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2610
2611 if (rtwdev->chip->chip_id == RTL8852A)
2612 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2613 B_AX_EN_CHKDSC_NO_RX_STUCK);
2614 }
2615
rtw89_pci_set_keep_reg(struct rtw89_dev * rtwdev)2616 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2617 {
2618 if (rtwdev->chip->chip_id == RTL8852C)
2619 return;
2620
2621 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2622 B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2623 }
2624
rtw89_pci_clr_idx_all_ax(struct rtw89_dev * rtwdev)2625 static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
2626 {
2627 const struct rtw89_pci_info *info = rtwdev->pci_info;
2628 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2629 u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2630 B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2631 B_AX_CLR_CH12_IDX;
2632 u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2633 u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2634
2635 if (chip_id == RTL8852A || chip_id == RTL8852C)
2636 val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2637 B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2638 /* clear DMA indexes */
2639 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2640 if (chip_id == RTL8852A || chip_id == RTL8852C)
2641 rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2642 B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2643 rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2644 B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2645 }
2646
rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev * rtwdev)2647 static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2648 {
2649 const struct rtw89_pci_info *info = rtwdev->pci_info;
2650 u32 ret, check, dma_busy;
2651 u32 dma_busy1 = info->dma_busy1.addr;
2652 u32 dma_busy2 = info->dma_busy2_reg;
2653
2654 check = info->dma_busy1.mask;
2655
2656 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2657 10, 100, false, rtwdev, dma_busy1);
2658 if (ret)
2659 return ret;
2660
2661 if (!dma_busy2)
2662 return 0;
2663
2664 check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2665
2666 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2667 10, 100, false, rtwdev, dma_busy2);
2668 if (ret)
2669 return ret;
2670
2671 return 0;
2672 }
2673
rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev * rtwdev)2674 static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2675 {
2676 const struct rtw89_pci_info *info = rtwdev->pci_info;
2677 u32 ret, check, dma_busy;
2678 u32 dma_busy3 = info->dma_busy3_reg;
2679
2680 check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2681
2682 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2683 10, 100, false, rtwdev, dma_busy3);
2684 if (ret)
2685 return ret;
2686
2687 return 0;
2688 }
2689
rtw89_pci_poll_dma_all_idle(struct rtw89_dev * rtwdev)2690 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2691 {
2692 u32 ret;
2693
2694 ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev);
2695 if (ret) {
2696 rtw89_err(rtwdev, "txdma ch busy\n");
2697 return ret;
2698 }
2699
2700 ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev);
2701 if (ret) {
2702 rtw89_err(rtwdev, "rxdma ch busy\n");
2703 return ret;
2704 }
2705
2706 return 0;
2707 }
2708
rtw89_pci_mode_op(struct rtw89_dev * rtwdev)2709 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2710 {
2711 const struct rtw89_pci_info *info = rtwdev->pci_info;
2712 enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
2713 enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
2714 enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
2715 enum mac_ax_tag_mode tag_mode = info->tag_mode;
2716 enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
2717 enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
2718 enum mac_ax_tx_burst tx_burst = info->tx_burst;
2719 enum mac_ax_rx_burst rx_burst = info->rx_burst;
2720 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2721 u8 cv = rtwdev->hal.cv;
2722 u32 val32;
2723
2724 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2725 if (chip_id == RTL8852A && cv == CHIP_CBV)
2726 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2727 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2728 if (chip_id == RTL8852A || chip_id == RTL8852B)
2729 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2730 }
2731
2732 if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
2733 if (chip_id == RTL8852A && cv == CHIP_CBV)
2734 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2735 } else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
2736 if (chip_id == RTL8852A || chip_id == RTL8852B)
2737 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2738 }
2739
2740 if (rxbd_mode == MAC_AX_RXBD_PKT) {
2741 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2742 } else if (rxbd_mode == MAC_AX_RXBD_SEP) {
2743 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2744
2745 if (chip_id == RTL8852A || chip_id == RTL8852B)
2746 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2747 B_AX_PCIE_RX_APPLEN_MASK, 0);
2748 }
2749
2750 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2751 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2752 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2753 } else if (chip_id == RTL8852C) {
2754 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2755 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2756 }
2757
2758 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2759 if (tag_mode == MAC_AX_TAG_SGL) {
2760 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2761 ~B_AX_LATENCY_CONTROL;
2762 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2763 } else if (tag_mode == MAC_AX_TAG_MULTI) {
2764 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2765 B_AX_LATENCY_CONTROL;
2766 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2767 }
2768 }
2769
2770 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2771 info->multi_tag_num);
2772
2773 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2774 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2775 wd_dma_idle_intvl);
2776 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2777 wd_dma_act_intvl);
2778 } else if (chip_id == RTL8852C) {
2779 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2780 wd_dma_idle_intvl);
2781 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2782 wd_dma_act_intvl);
2783 }
2784
2785 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2786 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2787 B_AX_HOST_ADDR_INFO_8B_SEL);
2788 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2789 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2790 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2791 B_AX_HOST_ADDR_INFO_8B_SEL);
2792 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2793 }
2794
2795 return 0;
2796 }
2797
rtw89_pci_ops_deinit(struct rtw89_dev * rtwdev)2798 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2799 {
2800 const struct rtw89_pci_info *info = rtwdev->pci_info;
2801
2802 if (rtwdev->chip->chip_id == RTL8852A) {
2803 /* ltr sw trigger */
2804 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2805 }
2806 info->ltr_set(rtwdev, false);
2807 rtw89_pci_ctrl_dma_all(rtwdev, false);
2808 rtw89_pci_clr_idx_all(rtwdev);
2809
2810 return 0;
2811 }
2812
rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev * rtwdev)2813 static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
2814 {
2815 const struct rtw89_pci_info *info = rtwdev->pci_info;
2816 int ret;
2817
2818 rtw89_pci_ber(rtwdev);
2819 rtw89_pci_rxdma_prefth(rtwdev);
2820 rtw89_pci_l1off_pwroff(rtwdev);
2821 rtw89_pci_deglitch_setting(rtwdev);
2822 ret = rtw89_pci_l2_rxen_lat(rtwdev);
2823 if (ret) {
2824 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2825 return ret;
2826 }
2827
2828 rtw89_pci_aphy_pwrcut(rtwdev);
2829 rtw89_pci_hci_ldo(rtwdev);
2830 rtw89_pci_dphy_delay(rtwdev);
2831
2832 ret = rtw89_pci_autok_x(rtwdev);
2833 if (ret) {
2834 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2835 return ret;
2836 }
2837
2838 ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2839 if (ret) {
2840 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2841 return ret;
2842 }
2843
2844 rtw89_pci_power_wake(rtwdev, true);
2845 rtw89_pci_autoload_hang(rtwdev);
2846 rtw89_pci_l12_vmain(rtwdev);
2847 rtw89_pci_gen2_force_ib(rtwdev);
2848 rtw89_pci_l1_ent_lat(rtwdev);
2849 rtw89_pci_wd_exit_l1(rtwdev);
2850 rtw89_pci_set_sic(rtwdev);
2851 rtw89_pci_set_lbc(rtwdev);
2852 rtw89_pci_set_io_rcy(rtwdev);
2853 rtw89_pci_set_dbg(rtwdev);
2854 rtw89_pci_set_keep_reg(rtwdev);
2855
2856 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2857
2858 /* stop DMA activities */
2859 rtw89_pci_ctrl_dma_all(rtwdev, false);
2860
2861 ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2862 if (ret) {
2863 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2864 return ret;
2865 }
2866
2867 rtw89_pci_clr_idx_all(rtwdev);
2868 rtw89_pci_mode_op(rtwdev);
2869
2870 /* fill TRX BD indexes */
2871 rtw89_pci_ops_reset(rtwdev);
2872
2873 ret = rtw89_pci_rst_bdram_ax(rtwdev);
2874 if (ret) {
2875 rtw89_warn(rtwdev, "reset bdram busy\n");
2876 return ret;
2877 }
2878
2879 /* disable all channels except to FW CMD channel to download firmware */
2880 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, false);
2881 rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, true);
2882
2883 /* start DMA activities */
2884 rtw89_pci_ctrl_dma_all(rtwdev, true);
2885
2886 return 0;
2887 }
2888
rtw89_pci_ltr_set(struct rtw89_dev * rtwdev,bool en)2889 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2890 {
2891 u32 val;
2892
2893 if (!en)
2894 return 0;
2895
2896 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2897 if (rtw89_pci_ltr_is_err_reg_val(val))
2898 return -EINVAL;
2899 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2900 if (rtw89_pci_ltr_is_err_reg_val(val))
2901 return -EINVAL;
2902 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2903 if (rtw89_pci_ltr_is_err_reg_val(val))
2904 return -EINVAL;
2905 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2906 if (rtw89_pci_ltr_is_err_reg_val(val))
2907 return -EINVAL;
2908
2909 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2910 B_AX_LTR_WD_NOEMP_CHK);
2911 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2912 PCI_LTR_SPC_500US);
2913 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2914 PCI_LTR_IDLE_TIMER_3_2MS);
2915 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2916 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2917 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
2918 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
2919
2920 return 0;
2921 }
2922 EXPORT_SYMBOL(rtw89_pci_ltr_set);
2923
rtw89_pci_ltr_set_v1(struct rtw89_dev * rtwdev,bool en)2924 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
2925 {
2926 u32 dec_ctrl;
2927 u32 val32;
2928
2929 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2930 if (rtw89_pci_ltr_is_err_reg_val(val32))
2931 return -EINVAL;
2932 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2933 if (rtw89_pci_ltr_is_err_reg_val(val32))
2934 return -EINVAL;
2935 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
2936 if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
2937 return -EINVAL;
2938 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
2939 if (rtw89_pci_ltr_is_err_reg_val(val32))
2940 return -EINVAL;
2941 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
2942 if (rtw89_pci_ltr_is_err_reg_val(val32))
2943 return -EINVAL;
2944
2945 if (!en) {
2946 dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
2947 dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
2948 B_AX_LTR_REQ_DRV;
2949 } else {
2950 dec_ctrl |= B_AX_LTR_HW_DEC_EN;
2951 }
2952
2953 dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
2954 dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
2955
2956 if (en)
2957 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
2958 B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
2959 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2960 PCI_LTR_IDLE_TIMER_3_2MS);
2961 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2962 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2963 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
2964 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
2965 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
2966
2967 return 0;
2968 }
2969 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
2970
rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev * rtwdev)2971 static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
2972 {
2973 const struct rtw89_pci_info *info = rtwdev->pci_info;
2974 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2975 int ret;
2976
2977 ret = info->ltr_set(rtwdev, true);
2978 if (ret) {
2979 rtw89_err(rtwdev, "pci ltr set fail\n");
2980 return ret;
2981 }
2982 if (chip_id == RTL8852A) {
2983 /* ltr sw trigger */
2984 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
2985 }
2986 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2987 /* ADDR info 8-byte mode */
2988 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2989 B_AX_HOST_ADDR_INFO_8B_SEL);
2990 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2991 }
2992
2993 /* enable DMA for all queues */
2994 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, true);
2995
2996 /* Release PCI IO */
2997 rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
2998 B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
2999
3000 return 0;
3001 }
3002
rtw89_pci_claim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3003 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
3004 struct pci_dev *pdev)
3005 {
3006 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3007 int ret;
3008
3009 ret = pci_enable_device(pdev);
3010 if (ret) {
3011 rtw89_err(rtwdev, "failed to enable pci device\n");
3012 return ret;
3013 }
3014
3015 pci_set_master(pdev);
3016 pci_set_drvdata(pdev, rtwdev->hw);
3017
3018 rtwpci->pdev = pdev;
3019
3020 return 0;
3021 }
3022
rtw89_pci_declaim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3023 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
3024 struct pci_dev *pdev)
3025 {
3026 pci_disable_device(pdev);
3027 }
3028
rtw89_pci_chip_is_manual_dac(struct rtw89_dev * rtwdev)3029 static bool rtw89_pci_chip_is_manual_dac(struct rtw89_dev *rtwdev)
3030 {
3031 const struct rtw89_chip_info *chip = rtwdev->chip;
3032
3033 switch (chip->chip_id) {
3034 case RTL8852A:
3035 case RTL8852B:
3036 case RTL8851B:
3037 case RTL8852BT:
3038 return true;
3039 default:
3040 return false;
3041 }
3042 }
3043
rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev * rtwdev)3044 static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev)
3045 {
3046 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3047 struct pci_dev *bridge = pci_upstream_bridge(rtwpci->pdev);
3048
3049 if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3050 return true;
3051
3052 if (!bridge)
3053 return false;
3054
3055 switch (bridge->vendor) {
3056 case PCI_VENDOR_ID_INTEL:
3057 return true;
3058 case PCI_VENDOR_ID_ASMEDIA:
3059 if (bridge->device == 0x2806)
3060 return true;
3061 break;
3062 }
3063
3064 return false;
3065 }
3066
rtw89_pci_cfg_dac(struct rtw89_dev * rtwdev)3067 static void rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev)
3068 {
3069 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3070
3071 if (!rtwpci->enable_dac)
3072 return;
3073
3074 if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3075 return;
3076
3077 rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, RTW89_PCIE_BIT_EN_64BITS);
3078 }
3079
rtw89_pci_setup_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3080 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
3081 struct pci_dev *pdev)
3082 {
3083 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3084 unsigned long resource_len;
3085 u8 bar_id = 2;
3086 int ret;
3087
3088 ret = pci_request_regions(pdev, KBUILD_MODNAME);
3089 if (ret) {
3090 rtw89_err(rtwdev, "failed to request pci regions\n");
3091 goto err;
3092 }
3093
3094 if (!rtw89_pci_is_dac_compatible_bridge(rtwdev))
3095 goto no_dac;
3096
3097 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36));
3098 if (!ret) {
3099 rtwpci->enable_dac = true;
3100 rtw89_pci_cfg_dac(rtwdev);
3101 } else {
3102 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3103 if (ret) {
3104 rtw89_err(rtwdev,
3105 "failed to set dma and consistent mask to 32/36-bit\n");
3106 goto err_release_regions;
3107 }
3108 }
3109 no_dac:
3110
3111 resource_len = pci_resource_len(pdev, bar_id);
3112 rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len);
3113 if (!rtwpci->mmap) {
3114 rtw89_err(rtwdev, "failed to map pci io\n");
3115 ret = -EIO;
3116 goto err_release_regions;
3117 }
3118
3119 return 0;
3120
3121 err_release_regions:
3122 pci_release_regions(pdev);
3123 err:
3124 return ret;
3125 }
3126
rtw89_pci_clear_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3127 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
3128 struct pci_dev *pdev)
3129 {
3130 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3131
3132 if (rtwpci->mmap) {
3133 pci_iounmap(pdev, rtwpci->mmap);
3134 pci_release_regions(pdev);
3135 }
3136 }
3137
rtw89_pci_free_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)3138 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
3139 struct pci_dev *pdev,
3140 struct rtw89_pci_tx_ring *tx_ring)
3141 {
3142 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
3143 u8 *head = wd_ring->head;
3144 dma_addr_t dma = wd_ring->dma;
3145 u32 page_size = wd_ring->page_size;
3146 u32 page_num = wd_ring->page_num;
3147 u32 ring_sz = page_size * page_num;
3148
3149 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3150 wd_ring->head = NULL;
3151 }
3152
rtw89_pci_free_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)3153 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
3154 struct pci_dev *pdev,
3155 struct rtw89_pci_tx_ring *tx_ring)
3156 {
3157 int ring_sz;
3158 u8 *head;
3159 dma_addr_t dma;
3160
3161 head = tx_ring->bd_ring.head;
3162 dma = tx_ring->bd_ring.dma;
3163 ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len;
3164 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3165
3166 tx_ring->bd_ring.head = NULL;
3167 }
3168
rtw89_pci_free_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3169 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
3170 struct pci_dev *pdev)
3171 {
3172 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3173 const struct rtw89_pci_info *info = rtwdev->pci_info;
3174 struct rtw89_pci_tx_ring *tx_ring;
3175 int i;
3176
3177 for (i = 0; i < RTW89_TXCH_NUM; i++) {
3178 if (info->tx_dma_ch_mask & BIT(i))
3179 continue;
3180 tx_ring = &rtwpci->tx_rings[i];
3181 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3182 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3183 }
3184 }
3185
rtw89_pci_free_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring)3186 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
3187 struct pci_dev *pdev,
3188 struct rtw89_pci_rx_ring *rx_ring)
3189 {
3190 struct rtw89_pci_rx_info *rx_info;
3191 struct sk_buff *skb;
3192 dma_addr_t dma;
3193 u32 buf_sz;
3194 u8 *head;
3195 int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len;
3196 int i;
3197
3198 buf_sz = rx_ring->buf_sz;
3199 for (i = 0; i < rx_ring->bd_ring.len; i++) {
3200 skb = rx_ring->buf[i];
3201 if (!skb)
3202 continue;
3203
3204 rx_info = RTW89_PCI_RX_SKB_CB(skb);
3205 dma = rx_info->dma;
3206 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3207 dev_kfree_skb(skb);
3208 rx_ring->buf[i] = NULL;
3209 }
3210
3211 head = rx_ring->bd_ring.head;
3212 dma = rx_ring->bd_ring.dma;
3213 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3214
3215 rx_ring->bd_ring.head = NULL;
3216 }
3217
rtw89_pci_free_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3218 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
3219 struct pci_dev *pdev)
3220 {
3221 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3222 struct rtw89_pci_rx_ring *rx_ring;
3223 int i;
3224
3225 for (i = 0; i < RTW89_RXCH_NUM; i++) {
3226 rx_ring = &rtwpci->rx_rings[i];
3227 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3228 }
3229 }
3230
rtw89_pci_free_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3231 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
3232 struct pci_dev *pdev)
3233 {
3234 rtw89_pci_free_rx_rings(rtwdev, pdev);
3235 rtw89_pci_free_tx_rings(rtwdev, pdev);
3236 }
3237
rtw89_pci_init_rx_bd(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb,int buf_sz,u32 idx)3238 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
3239 struct rtw89_pci_rx_ring *rx_ring,
3240 struct sk_buff *skb, int buf_sz, u32 idx)
3241 {
3242 struct rtw89_pci_rx_info *rx_info;
3243 struct rtw89_pci_rx_bd_32 *rx_bd;
3244 dma_addr_t dma;
3245
3246 if (!skb)
3247 return -EINVAL;
3248
3249 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
3250 if (dma_mapping_error(&pdev->dev, dma))
3251 return -EBUSY;
3252
3253 rx_info = RTW89_PCI_RX_SKB_CB(skb);
3254 rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
3255
3256 memset(rx_bd, 0, sizeof(*rx_bd));
3257 rx_bd->buf_size = cpu_to_le16(buf_sz);
3258 rx_bd->dma = cpu_to_le32(dma);
3259 rx_bd->opt = le16_encode_bits(upper_32_bits(dma), RTW89_PCI_RXBD_OPT_DMA_HI);
3260 rx_info->dma = dma;
3261
3262 return 0;
3263 }
3264
rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,enum rtw89_tx_channel txch)3265 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
3266 struct pci_dev *pdev,
3267 struct rtw89_pci_tx_ring *tx_ring,
3268 enum rtw89_tx_channel txch)
3269 {
3270 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
3271 struct rtw89_pci_tx_wd *txwd;
3272 dma_addr_t dma;
3273 dma_addr_t cur_paddr;
3274 u8 *head;
3275 u8 *cur_vaddr;
3276 u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
3277 u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
3278 u32 ring_sz = page_size * page_num;
3279 u32 page_offset;
3280 int i;
3281
3282 /* FWCMD queue doesn't use txwd as pages */
3283 if (txch == RTW89_TXCH_CH12)
3284 return 0;
3285
3286 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3287 if (!head)
3288 return -ENOMEM;
3289
3290 INIT_LIST_HEAD(&wd_ring->free_pages);
3291 wd_ring->head = head;
3292 wd_ring->dma = dma;
3293 wd_ring->page_size = page_size;
3294 wd_ring->page_num = page_num;
3295
3296 page_offset = 0;
3297 for (i = 0; i < page_num; i++) {
3298 txwd = &wd_ring->pages[i];
3299 cur_paddr = dma + page_offset;
3300 cur_vaddr = head + page_offset;
3301
3302 skb_queue_head_init(&txwd->queue);
3303 INIT_LIST_HEAD(&txwd->list);
3304 txwd->paddr = cur_paddr;
3305 txwd->vaddr = cur_vaddr;
3306 txwd->len = page_size;
3307 txwd->seq = i;
3308 rtw89_pci_enqueue_txwd(tx_ring, txwd);
3309
3310 page_offset += page_size;
3311 }
3312
3313 return 0;
3314 }
3315
rtw89_pci_alloc_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,u32 desc_size,u32 len,enum rtw89_tx_channel txch)3316 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
3317 struct pci_dev *pdev,
3318 struct rtw89_pci_tx_ring *tx_ring,
3319 u32 desc_size, u32 len,
3320 enum rtw89_tx_channel txch)
3321 {
3322 const struct rtw89_pci_ch_dma_addr *txch_addr;
3323 int ring_sz = desc_size * len;
3324 u8 *head;
3325 dma_addr_t dma;
3326 int ret;
3327
3328 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
3329 if (ret) {
3330 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
3331 goto err;
3332 }
3333
3334 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
3335 if (ret) {
3336 rtw89_err(rtwdev, "failed to get address of txch %d", txch);
3337 goto err_free_wd_ring;
3338 }
3339
3340 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3341 if (!head) {
3342 ret = -ENOMEM;
3343 goto err_free_wd_ring;
3344 }
3345
3346 INIT_LIST_HEAD(&tx_ring->busy_pages);
3347 tx_ring->bd_ring.head = head;
3348 tx_ring->bd_ring.dma = dma;
3349 tx_ring->bd_ring.len = len;
3350 tx_ring->bd_ring.desc_size = desc_size;
3351 tx_ring->bd_ring.addr = *txch_addr;
3352 tx_ring->bd_ring.wp = 0;
3353 tx_ring->bd_ring.rp = 0;
3354 tx_ring->txch = txch;
3355
3356 return 0;
3357
3358 err_free_wd_ring:
3359 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3360 err:
3361 return ret;
3362 }
3363
rtw89_pci_alloc_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3364 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
3365 struct pci_dev *pdev)
3366 {
3367 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3368 const struct rtw89_pci_info *info = rtwdev->pci_info;
3369 struct rtw89_pci_tx_ring *tx_ring;
3370 u32 desc_size;
3371 u32 len;
3372 u32 i, tx_allocated;
3373 int ret;
3374
3375 for (i = 0; i < RTW89_TXCH_NUM; i++) {
3376 if (info->tx_dma_ch_mask & BIT(i))
3377 continue;
3378 tx_ring = &rtwpci->tx_rings[i];
3379 desc_size = sizeof(struct rtw89_pci_tx_bd_32);
3380 len = RTW89_PCI_TXBD_NUM_MAX;
3381 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3382 desc_size, len, i);
3383 if (ret) {
3384 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3385 goto err_free;
3386 }
3387 }
3388
3389 return 0;
3390
3391 err_free:
3392 tx_allocated = i;
3393 for (i = 0; i < tx_allocated; i++) {
3394 tx_ring = &rtwpci->tx_rings[i];
3395 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3396 }
3397
3398 return ret;
3399 }
3400
rtw89_pci_alloc_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,u32 desc_size,u32 len,u32 rxch)3401 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3402 struct pci_dev *pdev,
3403 struct rtw89_pci_rx_ring *rx_ring,
3404 u32 desc_size, u32 len, u32 rxch)
3405 {
3406 const struct rtw89_pci_info *info = rtwdev->pci_info;
3407 const struct rtw89_pci_ch_dma_addr *rxch_addr;
3408 struct sk_buff *skb;
3409 u8 *head;
3410 dma_addr_t dma;
3411 int ring_sz = desc_size * len;
3412 int buf_sz = RTW89_PCI_RX_BUF_SIZE;
3413 int i, allocated;
3414 int ret;
3415
3416 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3417 if (ret) {
3418 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3419 return ret;
3420 }
3421
3422 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3423 if (!head) {
3424 ret = -ENOMEM;
3425 goto err;
3426 }
3427
3428 rx_ring->bd_ring.head = head;
3429 rx_ring->bd_ring.dma = dma;
3430 rx_ring->bd_ring.len = len;
3431 rx_ring->bd_ring.desc_size = desc_size;
3432 rx_ring->bd_ring.addr = *rxch_addr;
3433 if (info->rx_ring_eq_is_full)
3434 rx_ring->bd_ring.wp = len - 1;
3435 else
3436 rx_ring->bd_ring.wp = 0;
3437 rx_ring->bd_ring.rp = 0;
3438 rx_ring->buf_sz = buf_sz;
3439 rx_ring->diliver_skb = NULL;
3440 rx_ring->diliver_desc.ready = false;
3441 rx_ring->target_rx_tag = 0;
3442
3443 for (i = 0; i < len; i++) {
3444 skb = dev_alloc_skb(buf_sz);
3445 if (!skb) {
3446 ret = -ENOMEM;
3447 goto err_free;
3448 }
3449
3450 memset(skb->data, 0, buf_sz);
3451 rx_ring->buf[i] = skb;
3452 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3453 buf_sz, i);
3454 if (ret) {
3455 rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3456 dev_kfree_skb_any(skb);
3457 rx_ring->buf[i] = NULL;
3458 goto err_free;
3459 }
3460 }
3461
3462 return 0;
3463
3464 err_free:
3465 allocated = i;
3466 for (i = 0; i < allocated; i++) {
3467 skb = rx_ring->buf[i];
3468 if (!skb)
3469 continue;
3470 dma = *((dma_addr_t *)skb->cb);
3471 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3472 dev_kfree_skb(skb);
3473 rx_ring->buf[i] = NULL;
3474 }
3475
3476 head = rx_ring->bd_ring.head;
3477 dma = rx_ring->bd_ring.dma;
3478 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3479
3480 rx_ring->bd_ring.head = NULL;
3481 err:
3482 return ret;
3483 }
3484
rtw89_pci_alloc_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3485 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3486 struct pci_dev *pdev)
3487 {
3488 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3489 struct rtw89_pci_rx_ring *rx_ring;
3490 u32 desc_size;
3491 u32 len;
3492 int i, rx_allocated;
3493 int ret;
3494
3495 for (i = 0; i < RTW89_RXCH_NUM; i++) {
3496 rx_ring = &rtwpci->rx_rings[i];
3497 desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3498 len = RTW89_PCI_RXBD_NUM_MAX;
3499 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3500 desc_size, len, i);
3501 if (ret) {
3502 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3503 goto err_free;
3504 }
3505 }
3506
3507 return 0;
3508
3509 err_free:
3510 rx_allocated = i;
3511 for (i = 0; i < rx_allocated; i++) {
3512 rx_ring = &rtwpci->rx_rings[i];
3513 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3514 }
3515
3516 return ret;
3517 }
3518
rtw89_pci_alloc_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3519 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3520 struct pci_dev *pdev)
3521 {
3522 int ret;
3523
3524 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3525 if (ret) {
3526 rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3527 goto err;
3528 }
3529
3530 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3531 if (ret) {
3532 rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3533 goto err_free_tx_rings;
3534 }
3535
3536 return 0;
3537
3538 err_free_tx_rings:
3539 rtw89_pci_free_tx_rings(rtwdev, pdev);
3540 err:
3541 return ret;
3542 }
3543
rtw89_pci_h2c_init(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)3544 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3545 struct rtw89_pci *rtwpci)
3546 {
3547 skb_queue_head_init(&rtwpci->h2c_queue);
3548 skb_queue_head_init(&rtwpci->h2c_release_queue);
3549 }
3550
rtw89_pci_setup_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3551 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3552 struct pci_dev *pdev)
3553 {
3554 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3555 int ret;
3556
3557 ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3558 if (ret) {
3559 rtw89_err(rtwdev, "failed to setup pci mapping\n");
3560 goto err;
3561 }
3562
3563 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3564 if (ret) {
3565 rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3566 goto err_pci_unmap;
3567 }
3568
3569 rtw89_pci_h2c_init(rtwdev, rtwpci);
3570
3571 spin_lock_init(&rtwpci->irq_lock);
3572 spin_lock_init(&rtwpci->trx_lock);
3573
3574 return 0;
3575
3576 err_pci_unmap:
3577 rtw89_pci_clear_mapping(rtwdev, pdev);
3578 err:
3579 return ret;
3580 }
3581
rtw89_pci_clear_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3582 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3583 struct pci_dev *pdev)
3584 {
3585 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3586
3587 rtw89_pci_free_trx_rings(rtwdev, pdev);
3588 rtw89_pci_clear_mapping(rtwdev, pdev);
3589 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3590 skb_queue_len(&rtwpci->h2c_queue), true);
3591 }
3592
rtw89_pci_config_intr_mask(struct rtw89_dev * rtwdev)3593 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3594 {
3595 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3596 const struct rtw89_chip_info *chip = rtwdev->chip;
3597 u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN;
3598
3599 if (chip->chip_id == RTL8851B)
3600 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND;
3601
3602 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3603
3604 if (rtwpci->under_recovery) {
3605 rtwpci->intrs[0] = hs0isr_ind_int_en;
3606 rtwpci->intrs[1] = 0;
3607 } else {
3608 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3609 B_AX_RXDMA_INT_EN |
3610 B_AX_RXP1DMA_INT_EN |
3611 B_AX_RPQDMA_INT_EN |
3612 B_AX_RXDMA_STUCK_INT_EN |
3613 B_AX_RDU_INT_EN |
3614 B_AX_RPQBD_FULL_INT_EN |
3615 hs0isr_ind_int_en;
3616
3617 rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3618 }
3619 }
3620 EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3621
rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev * rtwdev)3622 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3623 {
3624 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3625
3626 rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
3627 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3628 rtwpci->intrs[0] = 0;
3629 rtwpci->intrs[1] = 0;
3630 }
3631
rtw89_pci_default_intr_mask_v1(struct rtw89_dev * rtwdev)3632 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3633 {
3634 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3635
3636 rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
3637 B_AX_HS1ISR_IND_INT_EN |
3638 B_AX_HS0ISR_IND_INT_EN;
3639 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3640 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3641 B_AX_RXDMA_INT_EN |
3642 B_AX_RXP1DMA_INT_EN |
3643 B_AX_RPQDMA_INT_EN |
3644 B_AX_RXDMA_STUCK_INT_EN |
3645 B_AX_RDU_INT_EN |
3646 B_AX_RPQBD_FULL_INT_EN;
3647 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3648 }
3649
rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev * rtwdev)3650 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3651 {
3652 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3653
3654 rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
3655 B_AX_HS0ISR_IND_INT_EN;
3656 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3657 rtwpci->intrs[0] = 0;
3658 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3659 }
3660
rtw89_pci_config_intr_mask_v1(struct rtw89_dev * rtwdev)3661 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3662 {
3663 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3664
3665 if (rtwpci->under_recovery)
3666 rtw89_pci_recovery_intr_mask_v1(rtwdev);
3667 else if (rtwpci->low_power)
3668 rtw89_pci_low_power_intr_mask_v1(rtwdev);
3669 else
3670 rtw89_pci_default_intr_mask_v1(rtwdev);
3671 }
3672 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
3673
rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev * rtwdev)3674 static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
3675 {
3676 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3677
3678 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
3679 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3680 rtwpci->intrs[0] = 0;
3681 rtwpci->intrs[1] = 0;
3682 }
3683
rtw89_pci_default_intr_mask_v2(struct rtw89_dev * rtwdev)3684 static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
3685 {
3686 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3687
3688 rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 |
3689 B_BE_HS0_IND_INT_EN0;
3690 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3691 rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 |
3692 B_BE_RDU_CH0_INT_IMR_V1;
3693 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3694 B_BE_PCIE_RX_RPQ0_IMR0_V1;
3695 }
3696
rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev * rtwdev)3697 static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
3698 {
3699 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3700
3701 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 |
3702 B_BE_HS1_IND_INT_EN0;
3703 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3704 rtwpci->intrs[0] = 0;
3705 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3706 B_BE_PCIE_RX_RPQ0_IMR0_V1;
3707 }
3708
rtw89_pci_config_intr_mask_v2(struct rtw89_dev * rtwdev)3709 void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
3710 {
3711 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3712
3713 if (rtwpci->under_recovery)
3714 rtw89_pci_recovery_intr_mask_v2(rtwdev);
3715 else if (rtwpci->low_power)
3716 rtw89_pci_low_power_intr_mask_v2(rtwdev);
3717 else
3718 rtw89_pci_default_intr_mask_v2(rtwdev);
3719 }
3720 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2);
3721
rtw89_pci_request_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3722 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3723 struct pci_dev *pdev)
3724 {
3725 unsigned long flags = 0;
3726 int ret;
3727
3728 flags |= PCI_IRQ_INTX | PCI_IRQ_MSI;
3729 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
3730 if (ret < 0) {
3731 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3732 goto err;
3733 }
3734
3735 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3736 rtw89_pci_interrupt_handler,
3737 rtw89_pci_interrupt_threadfn,
3738 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3739 if (ret) {
3740 rtw89_err(rtwdev, "failed to request threaded irq\n");
3741 goto err_free_vector;
3742 }
3743
3744 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3745
3746 return 0;
3747
3748 err_free_vector:
3749 pci_free_irq_vectors(pdev);
3750 err:
3751 return ret;
3752 }
3753
rtw89_pci_free_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3754 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3755 struct pci_dev *pdev)
3756 {
3757 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3758 pci_free_irq_vectors(pdev);
3759 }
3760
gray_code_to_bin(u16 gray_code,u32 bit_num)3761 static u16 gray_code_to_bin(u16 gray_code, u32 bit_num)
3762 {
3763 u16 bin = 0, gray_bit;
3764 u32 bit_idx;
3765
3766 for (bit_idx = 0; bit_idx < bit_num; bit_idx++) {
3767 gray_bit = (gray_code >> bit_idx) & 0x1;
3768 if (bit_num - bit_idx > 1)
3769 gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1;
3770 bin |= (gray_bit << bit_idx);
3771 }
3772
3773 return bin;
3774 }
3775
rtw89_pci_filter_out(struct rtw89_dev * rtwdev)3776 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3777 {
3778 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3779 struct pci_dev *pdev = rtwpci->pdev;
3780 u16 val16, filter_out_val;
3781 u32 val, phy_offset;
3782 int ret;
3783
3784 if (rtwdev->chip->chip_id != RTL8852C)
3785 return 0;
3786
3787 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3788 if (val == B_AX_ASPM_CTRL_L1)
3789 return 0;
3790
3791 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
3792 if (ret)
3793 return ret;
3794
3795 val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
3796 if (val == RTW89_PCIE_GEN1_SPEED) {
3797 phy_offset = R_RAC_DIRECT_OFFSET_G1;
3798 } else if (val == RTW89_PCIE_GEN2_SPEED) {
3799 phy_offset = R_RAC_DIRECT_OFFSET_G2;
3800 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3801 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3802 val16 | B_PCIE_BIT_PINOUT_DIS);
3803 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3804 val16 & ~B_PCIE_BIT_RD_SEL);
3805
3806 val16 = rtw89_read16_mask(rtwdev,
3807 phy_offset + RAC_ANA1F * RAC_MULT,
3808 FILTER_OUT_EQ_MASK);
3809 val16 = gray_code_to_bin(val16, hweight16(val16));
3810 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3811 RAC_MULT);
3812 filter_out_val &= ~REG_FILTER_OUT_MASK;
3813 filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
3814
3815 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3816 filter_out_val);
3817 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3818 B_BAC_EQ_SEL);
3819 rtw89_write16_set(rtwdev,
3820 R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
3821 B_PCIE_BIT_PSAVE);
3822 } else {
3823 return -EOPNOTSUPP;
3824 }
3825 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3826 B_PCIE_BIT_PSAVE);
3827
3828 return 0;
3829 }
3830
rtw89_pci_clkreq_set(struct rtw89_dev * rtwdev,bool enable)3831 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3832 {
3833 const struct rtw89_pci_info *info = rtwdev->pci_info;
3834 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3835
3836 if (rtw89_pci_disable_clkreq)
3837 return;
3838
3839 gen_def->clkreq_set(rtwdev, enable);
3840 }
3841
rtw89_pci_clkreq_set_ax(struct rtw89_dev * rtwdev,bool enable)3842 static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable)
3843 {
3844 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3845 int ret;
3846
3847 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3848 PCIE_CLKDLY_HW_30US);
3849 if (ret)
3850 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3851
3852 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3853 if (enable)
3854 ret = rtw89_pci_config_byte_set(rtwdev,
3855 RTW89_PCIE_L1_CTRL,
3856 RTW89_PCIE_BIT_CLK);
3857 else
3858 ret = rtw89_pci_config_byte_clr(rtwdev,
3859 RTW89_PCIE_L1_CTRL,
3860 RTW89_PCIE_BIT_CLK);
3861 if (ret)
3862 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3863 enable ? "set" : "unset", ret);
3864 } else if (chip_id == RTL8852C) {
3865 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3866 B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
3867 if (enable)
3868 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3869 B_AX_CLK_REQ_N);
3870 else
3871 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3872 B_AX_CLK_REQ_N);
3873 }
3874 }
3875
rtw89_pci_aspm_set(struct rtw89_dev * rtwdev,bool enable)3876 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3877 {
3878 const struct rtw89_pci_info *info = rtwdev->pci_info;
3879 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3880
3881 if (rtw89_pci_disable_aspm_l1)
3882 return;
3883
3884 gen_def->aspm_set(rtwdev, enable);
3885 }
3886
rtw89_pci_aspm_set_ax(struct rtw89_dev * rtwdev,bool enable)3887 static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable)
3888 {
3889 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3890 u8 value = 0;
3891 int ret;
3892
3893 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
3894 if (ret)
3895 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
3896
3897 u8p_replace_bits(&value, PCIE_L1DLY_16US, RTW89_L1DLY_MASK);
3898 u8p_replace_bits(&value, PCIE_L0SDLY_4US, RTW89_L0DLY_MASK);
3899
3900 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
3901 if (ret)
3902 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
3903
3904 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3905 if (enable)
3906 ret = rtw89_pci_config_byte_set(rtwdev,
3907 RTW89_PCIE_L1_CTRL,
3908 RTW89_PCIE_BIT_L1);
3909 else
3910 ret = rtw89_pci_config_byte_clr(rtwdev,
3911 RTW89_PCIE_L1_CTRL,
3912 RTW89_PCIE_BIT_L1);
3913 } else if (chip_id == RTL8852C) {
3914 if (enable)
3915 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3916 B_AX_ASPM_CTRL_L1);
3917 else
3918 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3919 B_AX_ASPM_CTRL_L1);
3920 }
3921 if (ret)
3922 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3923 enable ? "set" : "unset", ret);
3924 }
3925
rtw89_pci_recalc_int_mit(struct rtw89_dev * rtwdev)3926 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3927 {
3928 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
3929 const struct rtw89_pci_info *info = rtwdev->pci_info;
3930 struct rtw89_traffic_stats *stats = &rtwdev->stats;
3931 enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
3932 enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
3933 u32 val = 0;
3934
3935 if (rtwdev->scanning ||
3936 (tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH))
3937 goto out;
3938
3939 if (chip_gen == RTW89_CHIP_BE)
3940 val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN;
3941 else
3942 val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
3943 FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
3944 FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
3945 FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
3946
3947 out:
3948 rtw89_write32(rtwdev, info->mit_addr, val);
3949 }
3950
rtw89_pci_link_cfg(struct rtw89_dev * rtwdev)3951 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
3952 {
3953 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3954 struct pci_dev *pdev = rtwpci->pdev;
3955 u16 link_ctrl;
3956 int ret;
3957
3958 /* Though there is standard PCIE configuration space to set the
3959 * link control register, but by Realtek's design, driver should
3960 * check if host supports CLKREQ/ASPM to enable the HW module.
3961 *
3962 * These functions are implemented by two HW modules associated,
3963 * one is responsible to access PCIE configuration space to
3964 * follow the host settings, and another is in charge of doing
3965 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
3966 * the host does not support it, and due to some reasons or wrong
3967 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
3968 * loss if HW misbehaves on the link.
3969 *
3970 * Hence it's designed that driver should first check the PCIE
3971 * configuration space is sync'ed and enabled, then driver can turn
3972 * on the other module that is actually working on the mechanism.
3973 */
3974 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
3975 if (ret) {
3976 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
3977 return;
3978 }
3979
3980 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
3981 rtw89_pci_clkreq_set(rtwdev, true);
3982
3983 if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
3984 rtw89_pci_aspm_set(rtwdev, true);
3985 }
3986
rtw89_pci_l1ss_set(struct rtw89_dev * rtwdev,bool enable)3987 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
3988 {
3989 const struct rtw89_pci_info *info = rtwdev->pci_info;
3990 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3991
3992 if (rtw89_pci_disable_l1ss)
3993 return;
3994
3995 gen_def->l1ss_set(rtwdev, enable);
3996 }
3997
rtw89_pci_l1ss_set_ax(struct rtw89_dev * rtwdev,bool enable)3998 static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable)
3999 {
4000 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4001 int ret;
4002
4003 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4004 if (enable)
4005 ret = rtw89_pci_config_byte_set(rtwdev,
4006 RTW89_PCIE_TIMER_CTRL,
4007 RTW89_PCIE_BIT_L1SUB);
4008 else
4009 ret = rtw89_pci_config_byte_clr(rtwdev,
4010 RTW89_PCIE_TIMER_CTRL,
4011 RTW89_PCIE_BIT_L1SUB);
4012 if (ret)
4013 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
4014 enable ? "set" : "unset", ret);
4015 } else if (chip_id == RTL8852C) {
4016 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
4017 RTW89_PCIE_BIT_ASPM_L11 |
4018 RTW89_PCIE_BIT_PCI_L11);
4019 if (ret)
4020 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
4021 if (enable)
4022 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4023 B_AX_L1SUB_DISABLE);
4024 else
4025 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4026 B_AX_L1SUB_DISABLE);
4027 }
4028 }
4029
rtw89_pci_l1ss_cfg(struct rtw89_dev * rtwdev)4030 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
4031 {
4032 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4033 struct pci_dev *pdev = rtwpci->pdev;
4034 u32 l1ss_cap_ptr, l1ss_ctrl;
4035
4036 if (rtw89_pci_disable_l1ss)
4037 return;
4038
4039 l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
4040 if (!l1ss_cap_ptr)
4041 return;
4042
4043 pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl);
4044
4045 if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
4046 rtw89_pci_l1ss_set(rtwdev, true);
4047 }
4048
rtw89_pci_poll_io_idle_ax(struct rtw89_dev * rtwdev)4049 static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
4050 {
4051 int ret = 0;
4052 u32 sts;
4053 u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
4054
4055 ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
4056 10, 1000, false, rtwdev,
4057 R_AX_PCIE_DMA_BUSY1);
4058 if (ret) {
4059 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
4060 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
4061 return -EINVAL;
4062 }
4063 return ret;
4064 }
4065
rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev * rtwdev)4066 static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
4067 {
4068 u32 val;
4069 int ret;
4070
4071 if (rtwdev->chip->chip_id == RTL8852C)
4072 return 0;
4073
4074 rtw89_pci_ctrl_dma_all(rtwdev, false);
4075 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4076 if (ret) {
4077 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4078 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4079 "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
4080 R_AX_DBG_ERR_FLAG, val);
4081 if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
4082 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
4083 if (val & B_AX_RX_STUCK)
4084 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
4085 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4086 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4087 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4088 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4089 "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
4090 R_AX_DBG_ERR_FLAG, val);
4091 }
4092
4093 return ret;
4094 }
4095
rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev * rtwdev)4096 static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
4097 {
4098 u32 ret;
4099
4100 if (rtwdev->chip->chip_id == RTL8852C)
4101 return 0;
4102
4103 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
4104 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4105 rtw89_pci_clr_idx_all(rtwdev);
4106
4107 ret = rtw89_pci_rst_bdram_ax(rtwdev);
4108 if (ret)
4109 return ret;
4110
4111 rtw89_pci_ctrl_dma_all(rtwdev, true);
4112 return ret;
4113 }
4114
rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev * rtwdev,enum rtw89_lv1_rcvy_step step)4115 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
4116 enum rtw89_lv1_rcvy_step step)
4117 {
4118 const struct rtw89_pci_info *info = rtwdev->pci_info;
4119 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4120 int ret;
4121
4122 switch (step) {
4123 case RTW89_LV1_RCVY_STEP_1:
4124 ret = gen_def->lv1rst_stop_dma(rtwdev);
4125 if (ret)
4126 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
4127
4128 break;
4129
4130 case RTW89_LV1_RCVY_STEP_2:
4131 ret = gen_def->lv1rst_start_dma(rtwdev);
4132 if (ret)
4133 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
4134 break;
4135
4136 default:
4137 return -EINVAL;
4138 }
4139
4140 return ret;
4141 }
4142
rtw89_pci_ops_dump_err_status(struct rtw89_dev * rtwdev)4143 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
4144 {
4145 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
4146 return;
4147
4148 if (rtwdev->chip->chip_id == RTL8852C) {
4149 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4150 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1));
4151 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4152 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1));
4153 } else {
4154 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
4155 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
4156 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4157 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
4158 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4159 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
4160 }
4161 }
4162
rtw89_pci_napi_poll(struct napi_struct * napi,int budget)4163 static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
4164 {
4165 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
4166 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4167 const struct rtw89_pci_info *info = rtwdev->pci_info;
4168 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4169 unsigned long flags;
4170 int work_done;
4171
4172 rtwdev->napi_budget_countdown = budget;
4173
4174 rtw89_write32(rtwdev, gen_def->isr_clear_rpq.addr, gen_def->isr_clear_rpq.data);
4175 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4176 if (work_done == budget)
4177 return budget;
4178
4179 rtw89_write32(rtwdev, gen_def->isr_clear_rxq.addr, gen_def->isr_clear_rxq.data);
4180 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4181 if (work_done < budget && napi_complete_done(napi, work_done)) {
4182 spin_lock_irqsave(&rtwpci->irq_lock, flags);
4183 if (likely(rtwpci->running))
4184 rtw89_chip_enable_intr(rtwdev, rtwpci);
4185 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
4186 }
4187
4188 return work_done;
4189 }
4190
rtw89_pci_suspend(struct device * dev)4191 static int __maybe_unused rtw89_pci_suspend(struct device *dev)
4192 {
4193 struct ieee80211_hw *hw = dev_get_drvdata(dev);
4194 struct rtw89_dev *rtwdev = hw->priv;
4195 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4196
4197 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4198 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4199 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4200 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4201 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
4202 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
4203 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
4204 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
4205 } else {
4206 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4207 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
4208 }
4209
4210 return 0;
4211 }
4212
rtw89_pci_l2_hci_ldo(struct rtw89_dev * rtwdev)4213 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
4214 {
4215 if (rtwdev->chip->chip_id == RTL8852C)
4216 return;
4217
4218 /* Hardware need write the reg twice to ensure the setting work */
4219 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4220 RTW89_PCIE_BIT_CFG_RST_MSTATE);
4221 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4222 RTW89_PCIE_BIT_CFG_RST_MSTATE);
4223 }
4224
rtw89_pci_resume(struct device * dev)4225 static int __maybe_unused rtw89_pci_resume(struct device *dev)
4226 {
4227 struct ieee80211_hw *hw = dev_get_drvdata(dev);
4228 struct rtw89_dev *rtwdev = hw->priv;
4229 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4230
4231 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4232 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4233 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4234 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4235 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
4236 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
4237 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
4238 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
4239 } else {
4240 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4241 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
4242 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4243 B_AX_SEL_REQ_ENTR_L1);
4244 }
4245 rtw89_pci_l2_hci_ldo(rtwdev);
4246 rtw89_pci_disable_eq(rtwdev);
4247 rtw89_pci_cfg_dac(rtwdev);
4248 rtw89_pci_filter_out(rtwdev);
4249 rtw89_pci_link_cfg(rtwdev);
4250 rtw89_pci_l1ss_cfg(rtwdev);
4251
4252 return 0;
4253 }
4254
4255 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
4256 EXPORT_SYMBOL(rtw89_pm_ops);
4257
4258 const struct rtw89_pci_gen_def rtw89_pci_gen_ax = {
4259 .isr_rdu = B_AX_RDU_INT,
4260 .isr_halt_c2h = B_AX_HALT_C2H_INT_EN,
4261 .isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN,
4262 .isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT},
4263 .isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT |
4264 B_AX_RDU_INT},
4265
4266 .mac_pre_init = rtw89_pci_ops_mac_pre_init_ax,
4267 .mac_pre_deinit = NULL,
4268 .mac_post_init = rtw89_pci_ops_mac_post_init_ax,
4269
4270 .clr_idx_all = rtw89_pci_clr_idx_all_ax,
4271 .rst_bdram = rtw89_pci_rst_bdram_ax,
4272
4273 .lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax,
4274 .lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax,
4275
4276 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_ax,
4277 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_ax,
4278 .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle_ax,
4279
4280 .aspm_set = rtw89_pci_aspm_set_ax,
4281 .clkreq_set = rtw89_pci_clkreq_set_ax,
4282 .l1ss_set = rtw89_pci_l1ss_set_ax,
4283 };
4284 EXPORT_SYMBOL(rtw89_pci_gen_ax);
4285
4286 static const struct rtw89_hci_ops rtw89_pci_ops = {
4287 .tx_write = rtw89_pci_ops_tx_write,
4288 .tx_kick_off = rtw89_pci_ops_tx_kick_off,
4289 .flush_queues = rtw89_pci_ops_flush_queues,
4290 .reset = rtw89_pci_ops_reset,
4291 .start = rtw89_pci_ops_start,
4292 .stop = rtw89_pci_ops_stop,
4293 .pause = rtw89_pci_ops_pause,
4294 .switch_mode = rtw89_pci_ops_switch_mode,
4295 .recalc_int_mit = rtw89_pci_recalc_int_mit,
4296
4297 .read8 = rtw89_pci_ops_read8,
4298 .read16 = rtw89_pci_ops_read16,
4299 .read32 = rtw89_pci_ops_read32,
4300 .write8 = rtw89_pci_ops_write8,
4301 .write16 = rtw89_pci_ops_write16,
4302 .write32 = rtw89_pci_ops_write32,
4303
4304 .mac_pre_init = rtw89_pci_ops_mac_pre_init,
4305 .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit,
4306 .mac_post_init = rtw89_pci_ops_mac_post_init,
4307 .deinit = rtw89_pci_ops_deinit,
4308
4309 .check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
4310 .mac_lv1_rcvy = rtw89_pci_ops_mac_lv1_recovery,
4311 .dump_err_status = rtw89_pci_ops_dump_err_status,
4312 .napi_poll = rtw89_pci_napi_poll,
4313
4314 .recovery_start = rtw89_pci_ops_recovery_start,
4315 .recovery_complete = rtw89_pci_ops_recovery_complete,
4316
4317 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch,
4318 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch,
4319 .ctrl_trxhci = rtw89_pci_ctrl_dma_trx,
4320 .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle,
4321
4322 .clr_idx_all = rtw89_pci_clr_idx_all,
4323 .clear = rtw89_pci_clear_resource,
4324 .disable_intr = rtw89_pci_disable_intr_lock,
4325 .enable_intr = rtw89_pci_enable_intr_lock,
4326 .rst_bdram = rtw89_pci_reset_bdram,
4327 };
4328
rtw89_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)4329 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4330 {
4331 struct rtw89_dev *rtwdev;
4332 const struct rtw89_driver_info *info;
4333 const struct rtw89_pci_info *pci_info;
4334 int ret;
4335
4336 info = (const struct rtw89_driver_info *)id->driver_data;
4337
4338 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
4339 sizeof(struct rtw89_pci),
4340 info->chip);
4341 if (!rtwdev) {
4342 dev_err(&pdev->dev, "failed to allocate hw\n");
4343 return -ENOMEM;
4344 }
4345
4346 pci_info = info->bus.pci;
4347
4348 rtwdev->pci_info = info->bus.pci;
4349 rtwdev->hci.ops = &rtw89_pci_ops;
4350 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
4351 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
4352 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
4353
4354 rtw89_check_quirks(rtwdev, info->quirks);
4355
4356 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
4357
4358 ret = rtw89_core_init(rtwdev);
4359 if (ret) {
4360 rtw89_err(rtwdev, "failed to initialise core\n");
4361 goto err_release_hw;
4362 }
4363
4364 ret = rtw89_pci_claim_device(rtwdev, pdev);
4365 if (ret) {
4366 rtw89_err(rtwdev, "failed to claim pci device\n");
4367 goto err_core_deinit;
4368 }
4369
4370 ret = rtw89_pci_setup_resource(rtwdev, pdev);
4371 if (ret) {
4372 rtw89_err(rtwdev, "failed to setup pci resource\n");
4373 goto err_declaim_pci;
4374 }
4375
4376 ret = rtw89_chip_info_setup(rtwdev);
4377 if (ret) {
4378 rtw89_err(rtwdev, "failed to setup chip information\n");
4379 goto err_clear_resource;
4380 }
4381
4382 rtw89_pci_disable_eq(rtwdev);
4383 rtw89_pci_filter_out(rtwdev);
4384 rtw89_pci_link_cfg(rtwdev);
4385 rtw89_pci_l1ss_cfg(rtwdev);
4386
4387 ret = rtw89_core_napi_init(rtwdev);
4388 if (ret) {
4389 rtw89_err(rtwdev, "failed to init napi\n");
4390 goto err_clear_resource;
4391 }
4392
4393 ret = rtw89_pci_request_irq(rtwdev, pdev);
4394 if (ret) {
4395 rtw89_err(rtwdev, "failed to request pci irq\n");
4396 goto err_deinit_napi;
4397 }
4398
4399 ret = rtw89_core_register(rtwdev);
4400 if (ret) {
4401 rtw89_err(rtwdev, "failed to register core\n");
4402 goto err_free_irq;
4403 }
4404
4405 set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags);
4406
4407 return 0;
4408
4409 err_free_irq:
4410 rtw89_pci_free_irq(rtwdev, pdev);
4411 err_deinit_napi:
4412 rtw89_core_napi_deinit(rtwdev);
4413 err_clear_resource:
4414 rtw89_pci_clear_resource(rtwdev, pdev);
4415 err_declaim_pci:
4416 rtw89_pci_declaim_device(rtwdev, pdev);
4417 err_core_deinit:
4418 rtw89_core_deinit(rtwdev);
4419 err_release_hw:
4420 rtw89_free_ieee80211_hw(rtwdev);
4421
4422 return ret;
4423 }
4424 EXPORT_SYMBOL(rtw89_pci_probe);
4425
rtw89_pci_remove(struct pci_dev * pdev)4426 void rtw89_pci_remove(struct pci_dev *pdev)
4427 {
4428 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
4429 struct rtw89_dev *rtwdev;
4430
4431 rtwdev = hw->priv;
4432
4433 rtw89_pci_free_irq(rtwdev, pdev);
4434 rtw89_core_napi_deinit(rtwdev);
4435 rtw89_core_unregister(rtwdev);
4436 rtw89_pci_clear_resource(rtwdev, pdev);
4437 rtw89_pci_declaim_device(rtwdev, pdev);
4438 rtw89_core_deinit(rtwdev);
4439 rtw89_free_ieee80211_hw(rtwdev);
4440 }
4441 EXPORT_SYMBOL(rtw89_pci_remove);
4442
4443 MODULE_AUTHOR("Realtek Corporation");
4444 MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver");
4445 MODULE_LICENSE("Dual BSD/GPL");
4446