1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2020 Realtek Corporation
3 */
4
5 #if defined(__FreeBSD__)
6 #define LINUXKPI_PARAM_PREFIX rtw89_pci_
7 #endif
8
9 #include <linux/pci.h>
10 #if defined(__FreeBSD__)
11 #include <sys/rman.h>
12 #endif
13
14 #include "mac.h"
15 #include "pci.h"
16 #include "reg.h"
17 #include "ser.h"
18
19 static bool rtw89_pci_disable_clkreq;
20 static bool rtw89_pci_disable_aspm_l1;
21 static bool rtw89_pci_disable_l1ss;
22 module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
23 module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
24 module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
25 MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
26 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
27 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
28
rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev * rtwdev,u32 * phy_offset)29 static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev,
30 u32 *phy_offset)
31 {
32 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
33 struct pci_dev *pdev = rtwpci->pdev;
34 u32 val;
35 int ret;
36
37 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
38 if (ret)
39 return ret;
40
41 val = u32_get_bits(val, RTW89_BCFG_LINK_SPEED_MASK);
42 if (val == RTW89_PCIE_GEN1_SPEED) {
43 *phy_offset = R_RAC_DIRECT_OFFSET_G1;
44 } else if (val == RTW89_PCIE_GEN2_SPEED) {
45 *phy_offset = R_RAC_DIRECT_OFFSET_G2;
46 } else {
47 rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val);
48 return -EFAULT;
49 }
50
51 return 0;
52 }
53
rtw89_pci_rst_bdram_ax(struct rtw89_dev * rtwdev)54 static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
55 {
56 u32 val;
57 int ret;
58
59 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
60
61 ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
62 1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
63 rtwdev, R_AX_PCIE_INIT_CFG1);
64
65 return ret;
66 }
67
rtw89_pci_dma_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring,u32 cur_idx,bool tx)68 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
69 struct rtw89_pci_dma_ring *bd_ring,
70 u32 cur_idx, bool tx)
71 {
72 const struct rtw89_pci_info *info = rtwdev->pci_info;
73 u32 cnt, cur_rp, wp, rp, len;
74
75 rp = bd_ring->rp;
76 wp = bd_ring->wp;
77 len = bd_ring->len;
78
79 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
80 if (tx) {
81 cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
82 } else {
83 if (info->rx_ring_eq_is_full)
84 wp += 1;
85
86 cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
87 }
88
89 bd_ring->rp = cur_rp;
90
91 return cnt;
92 }
93
rtw89_pci_txbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)94 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
95 struct rtw89_pci_tx_ring *tx_ring)
96 {
97 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
98 u32 addr_idx = bd_ring->addr.idx;
99 u32 cnt, idx;
100
101 idx = rtw89_read32(rtwdev, addr_idx);
102 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
103
104 return cnt;
105 }
106
rtw89_pci_release_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,u32 cnt,bool release_all)107 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
108 struct rtw89_pci *rtwpci,
109 u32 cnt, bool release_all)
110 {
111 struct rtw89_pci_tx_data *tx_data;
112 struct sk_buff *skb;
113 u32 qlen;
114
115 while (cnt--) {
116 skb = skb_dequeue(&rtwpci->h2c_queue);
117 if (!skb) {
118 rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
119 return;
120 }
121 skb_queue_tail(&rtwpci->h2c_release_queue, skb);
122 }
123
124 qlen = skb_queue_len(&rtwpci->h2c_release_queue);
125 if (!release_all)
126 qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
127
128 while (qlen--) {
129 skb = skb_dequeue(&rtwpci->h2c_release_queue);
130 if (!skb) {
131 rtw89_err(rtwdev, "failed to release fwcmd\n");
132 return;
133 }
134 tx_data = RTW89_PCI_TX_SKB_CB(skb);
135 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
136 DMA_TO_DEVICE);
137 dev_kfree_skb_any(skb);
138 }
139 }
140
rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)141 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
142 struct rtw89_pci *rtwpci)
143 {
144 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[RTW89_TXCH_CH12];
145 u32 cnt;
146
147 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
148 if (!cnt)
149 return;
150 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
151 }
152
rtw89_pci_rxbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)153 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
154 struct rtw89_pci_rx_ring *rx_ring)
155 {
156 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
157 u32 addr_idx = bd_ring->addr.idx;
158 u32 cnt, idx;
159
160 idx = rtw89_read32(rtwdev, addr_idx);
161 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
162
163 return cnt;
164 }
165
rtw89_pci_sync_skb_for_cpu(struct rtw89_dev * rtwdev,struct sk_buff * skb)166 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
167 struct sk_buff *skb)
168 {
169 struct rtw89_pci_rx_info *rx_info;
170 dma_addr_t dma;
171
172 rx_info = RTW89_PCI_RX_SKB_CB(skb);
173 dma = rx_info->dma;
174 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
175 DMA_FROM_DEVICE);
176 }
177
rtw89_pci_sync_skb_for_device(struct rtw89_dev * rtwdev,struct sk_buff * skb)178 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
179 struct sk_buff *skb)
180 {
181 struct rtw89_pci_rx_info *rx_info;
182 dma_addr_t dma;
183
184 rx_info = RTW89_PCI_RX_SKB_CB(skb);
185 dma = rx_info->dma;
186 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
187 DMA_FROM_DEVICE);
188 }
189
rtw89_pci_rxbd_info_update(struct rtw89_dev * rtwdev,struct sk_buff * skb)190 static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
191 struct sk_buff *skb)
192 {
193 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
194 struct rtw89_pci_rxbd_info *rxbd_info;
195 __le32 info;
196
197 rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
198 info = rxbd_info->dword;
199
200 rx_info->fs = le32_get_bits(info, RTW89_PCI_RXBD_FS);
201 rx_info->ls = le32_get_bits(info, RTW89_PCI_RXBD_LS);
202 rx_info->len = le32_get_bits(info, RTW89_PCI_RXBD_WRITE_SIZE);
203 rx_info->tag = le32_get_bits(info, RTW89_PCI_RXBD_TAG);
204 }
205
rtw89_pci_validate_rx_tag(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb)206 static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev,
207 struct rtw89_pci_rx_ring *rx_ring,
208 struct sk_buff *skb)
209 {
210 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
211 const struct rtw89_pci_info *info = rtwdev->pci_info;
212 u32 target_rx_tag;
213
214 if (!info->check_rx_tag)
215 return 0;
216
217 /* valid range is 1 ~ 0x1FFF */
218 if (rx_ring->target_rx_tag == 0)
219 target_rx_tag = 1;
220 else
221 target_rx_tag = rx_ring->target_rx_tag;
222
223 if (rx_info->tag != target_rx_tag) {
224 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "mismatch RX tag 0x%x 0x%x\n",
225 rx_info->tag, target_rx_tag);
226 return -EAGAIN;
227 }
228
229 return 0;
230 }
231
232 static
rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb)233 int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev,
234 struct rtw89_pci_rx_ring *rx_ring,
235 struct sk_buff *skb)
236 {
237 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
238 int rx_tag_retry = 1000;
239 int ret;
240
241 do {
242 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
243 rtw89_pci_rxbd_info_update(rtwdev, skb);
244
245 ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb);
246 if (ret != -EAGAIN)
247 break;
248 } while (rx_tag_retry--);
249
250 /* update target rx_tag for next RX */
251 rx_ring->target_rx_tag = rx_info->tag + 1;
252
253 return ret;
254 }
255
rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev * rtwdev,bool enable)256 static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable)
257 {
258 const struct rtw89_pci_info *info = rtwdev->pci_info;
259 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
260 const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
261
262 if (enable) {
263 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
264 if (dma_stop2->addr)
265 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
266 } else {
267 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
268 if (dma_stop2->addr)
269 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
270 }
271 }
272
rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev * rtwdev,bool enable)273 static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable)
274 {
275 const struct rtw89_pci_info *info = rtwdev->pci_info;
276 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
277
278 if (enable)
279 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
280 else
281 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
282 }
283
284 static bool
rtw89_skb_put_rx_data(struct rtw89_dev * rtwdev,bool fs,bool ls,struct sk_buff * new,const struct sk_buff * skb,u32 offset,const struct rtw89_pci_rx_info * rx_info,const struct rtw89_rx_desc_info * desc_info)285 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
286 struct sk_buff *new,
287 const struct sk_buff *skb, u32 offset,
288 const struct rtw89_pci_rx_info *rx_info,
289 const struct rtw89_rx_desc_info *desc_info)
290 {
291 u32 copy_len = rx_info->len - offset;
292
293 if (unlikely(skb_tailroom(new) < copy_len)) {
294 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
295 "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
296 rx_info->len, desc_info->pkt_size, offset, fs, ls);
297 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
298 skb->data, rx_info->len);
299 /* length of a single segment skb is desc_info->pkt_size */
300 if (fs && ls) {
301 copy_len = desc_info->pkt_size;
302 } else {
303 rtw89_info(rtwdev, "drop rx data due to invalid length\n");
304 return false;
305 }
306 }
307
308 skb_put_data(new, skb->data + offset, copy_len);
309
310 return true;
311 }
312
rtw89_pci_get_rx_skb_idx(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring)313 static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
314 struct rtw89_pci_dma_ring *bd_ring)
315 {
316 const struct rtw89_pci_info *info = rtwdev->pci_info;
317 u32 wp = bd_ring->wp;
318
319 if (!info->rx_ring_eq_is_full)
320 return wp;
321
322 if (++wp >= bd_ring->len)
323 wp = 0;
324
325 return wp;
326 }
327
rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)328 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
329 struct rtw89_pci_rx_ring *rx_ring)
330 {
331 struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
332 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
333 const struct rtw89_pci_info *info = rtwdev->pci_info;
334 struct sk_buff *new = rx_ring->diliver_skb;
335 struct rtw89_pci_rx_info *rx_info;
336 struct sk_buff *skb;
337 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
338 u32 skb_idx;
339 u32 offset;
340 u32 cnt = 1;
341 bool fs, ls;
342 int ret;
343
344 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
345 skb = rx_ring->buf[skb_idx];
346
347 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
348 if (ret) {
349 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
350 bd_ring->wp, ret);
351 goto err_sync_device;
352 }
353
354 rx_info = RTW89_PCI_RX_SKB_CB(skb);
355 fs = info->no_rxbd_fs ? !new : rx_info->fs;
356 ls = rx_info->ls;
357
358 if (unlikely(!fs || !ls))
359 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
360 "unexpected fs/ls=%d/%d tag=%u len=%u new->len=%u\n",
361 fs, ls, rx_info->tag, rx_info->len, new ? new->len : 0);
362
363 if (fs) {
364 if (new) {
365 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
366 "skb should not be ready before first segment start\n");
367 goto err_sync_device;
368 }
369 if (desc_info->ready) {
370 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
371 goto err_sync_device;
372 }
373
374 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
375
376 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
377 if (!new)
378 goto err_sync_device;
379
380 rx_ring->diliver_skb = new;
381
382 /* first segment has RX desc */
383 offset = desc_info->offset + desc_info->rxd_len;
384 } else {
385 offset = sizeof(struct rtw89_pci_rxbd_info);
386 if (!new) {
387 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
388 goto err_sync_device;
389 }
390 }
391 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
392 goto err_sync_device;
393 rtw89_pci_sync_skb_for_device(rtwdev, skb);
394 rtw89_pci_rxbd_increase(rx_ring, 1);
395
396 if (!desc_info->ready) {
397 rtw89_warn(rtwdev, "no rx desc information\n");
398 goto err_free_resource;
399 }
400 if (ls) {
401 rtw89_core_rx(rtwdev, desc_info, new);
402 rx_ring->diliver_skb = NULL;
403 desc_info->ready = false;
404 }
405
406 return cnt;
407
408 err_sync_device:
409 rtw89_pci_sync_skb_for_device(rtwdev, skb);
410 rtw89_pci_rxbd_increase(rx_ring, 1);
411 err_free_resource:
412 if (new)
413 dev_kfree_skb_any(new);
414 rx_ring->diliver_skb = NULL;
415 desc_info->ready = false;
416
417 return cnt;
418 }
419
rtw89_pci_rxbd_deliver(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)420 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
421 struct rtw89_pci_rx_ring *rx_ring,
422 u32 cnt)
423 {
424 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
425 u32 rx_cnt;
426
427 while (cnt && rtwdev->napi_budget_countdown > 0) {
428 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
429 if (!rx_cnt) {
430 rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
431
432 /* skip the rest RXBD bufs */
433 rtw89_pci_rxbd_increase(rx_ring, cnt);
434 break;
435 }
436
437 cnt -= rx_cnt;
438 }
439
440 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
441 }
442
rtw89_pci_poll_rxq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)443 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
444 struct rtw89_pci *rtwpci, int budget)
445 {
446 struct rtw89_pci_rx_ring *rx_ring;
447 int countdown = rtwdev->napi_budget_countdown;
448 u32 cnt;
449
450 rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RXQ];
451
452 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
453 if (!cnt)
454 return 0;
455
456 cnt = min_t(u32, budget, cnt);
457
458 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
459
460 /* In case of flushing pending SKBs, the countdown may exceed. */
461 if (rtwdev->napi_budget_countdown <= 0)
462 return budget;
463
464 return budget - countdown;
465 }
466
rtw89_pci_tx_status(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct sk_buff * skb,u8 tx_status)467 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
468 struct rtw89_pci_tx_ring *tx_ring,
469 struct sk_buff *skb, u8 tx_status)
470 {
471 struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
472 struct ieee80211_tx_info *info;
473
474 if (rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status))
475 return;
476
477 info = IEEE80211_SKB_CB(skb);
478 ieee80211_tx_info_clear_status(info);
479
480 if (info->flags & IEEE80211_TX_CTL_NO_ACK)
481 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
482 if (tx_status == RTW89_TX_DONE) {
483 info->flags |= IEEE80211_TX_STAT_ACK;
484 tx_ring->tx_acked++;
485 } else {
486 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
487 rtw89_debug(rtwdev, RTW89_DBG_FW,
488 "failed to TX of status %x\n", tx_status);
489 switch (tx_status) {
490 case RTW89_TX_RETRY_LIMIT:
491 tx_ring->tx_retry_lmt++;
492 break;
493 case RTW89_TX_LIFE_TIME:
494 tx_ring->tx_life_time++;
495 break;
496 case RTW89_TX_MACID_DROP:
497 tx_ring->tx_mac_id_drop++;
498 break;
499 default:
500 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
501 break;
502 }
503 }
504
505 ieee80211_tx_status_ni(rtwdev->hw, skb);
506 }
507
rtw89_pci_reclaim_txbd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)508 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
509 {
510 struct rtw89_pci_tx_wd *txwd;
511 u32 cnt;
512
513 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
514 while (cnt--) {
515 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
516 if (!txwd) {
517 rtw89_warn(rtwdev, "No busy txwd pages available\n");
518 break;
519 }
520
521 list_del_init(&txwd->list);
522
523 /* this skb has been freed by RPP */
524 if (skb_queue_len(&txwd->queue) == 0)
525 rtw89_pci_enqueue_txwd(tx_ring, txwd);
526 }
527 }
528
rtw89_pci_release_busy_txwd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)529 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
530 struct rtw89_pci_tx_ring *tx_ring)
531 {
532 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
533 struct rtw89_pci_tx_wd *txwd;
534 int i;
535
536 for (i = 0; i < wd_ring->page_num; i++) {
537 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
538 if (!txwd)
539 break;
540
541 list_del_init(&txwd->list);
542 }
543 }
544
rtw89_pci_release_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,u16 seq,u8 tx_status)545 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
546 struct rtw89_pci_tx_ring *tx_ring,
547 struct rtw89_pci_tx_wd *txwd, u16 seq,
548 u8 tx_status)
549 {
550 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
551 struct rtw89_pci_tx_data *tx_data;
552 struct sk_buff *skb, *tmp;
553 u8 txch = tx_ring->txch;
554
555 if (!list_empty(&txwd->list)) {
556 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
557 /* In low power mode, RPP can receive before updating of TX BD.
558 * In normal mode, it should not happen so give it a warning.
559 */
560 if (!rtwpci->low_power && !list_empty(&txwd->list))
561 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
562 txch, seq);
563 }
564
565 skb_queue_walk_safe(&txwd->queue, skb, tmp) {
566 skb_unlink(skb, &txwd->queue);
567
568 tx_data = RTW89_PCI_TX_SKB_CB(skb);
569 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
570 DMA_TO_DEVICE);
571
572 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
573 }
574
575 if (list_empty(&txwd->list))
576 rtw89_pci_enqueue_txwd(tx_ring, txwd);
577 }
578
rtw89_pci_parse_rpp(struct rtw89_dev * rtwdev,void * _rpp,struct rtw89_pci_rpp_info * rpp_info)579 void rtw89_pci_parse_rpp(struct rtw89_dev *rtwdev, void *_rpp,
580 struct rtw89_pci_rpp_info *rpp_info)
581 {
582 const struct rtw89_pci_rpp_fmt *rpp = _rpp;
583
584 rpp_info->seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ);
585 rpp_info->qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL);
586 rpp_info->tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS);
587 rpp_info->txch = rtw89_chip_get_ch_dma(rtwdev, rpp_info->qsel);
588 }
589 EXPORT_SYMBOL(rtw89_pci_parse_rpp);
590
rtw89_pci_parse_rpp_v1(struct rtw89_dev * rtwdev,void * _rpp,struct rtw89_pci_rpp_info * rpp_info)591 void rtw89_pci_parse_rpp_v1(struct rtw89_dev *rtwdev, void *_rpp,
592 struct rtw89_pci_rpp_info *rpp_info)
593 {
594 const struct rtw89_pci_rpp_fmt_v1 *rpp = _rpp;
595
596 rpp_info->seq = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_PCIE_SEQ_V1_MASK);
597 rpp_info->qsel = le32_get_bits(rpp->w1, RTW89_PCI_RPP_W1_QSEL_V1_MASK);
598 rpp_info->tx_status = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_TX_STATUS_V1_MASK);
599 rpp_info->txch = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_DMA_CH_MASK);
600 }
601 EXPORT_SYMBOL(rtw89_pci_parse_rpp_v1);
602
rtw89_pci_release_rpp(struct rtw89_dev * rtwdev,void * rpp)603 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, void *rpp)
604 {
605 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
606 const struct rtw89_pci_info *info = rtwdev->pci_info;
607 struct rtw89_pci_rpp_info rpp_info = {};
608 struct rtw89_pci_tx_wd_ring *wd_ring;
609 struct rtw89_pci_tx_ring *tx_ring;
610 struct rtw89_pci_tx_wd *txwd;
611
612 info->parse_rpp(rtwdev, rpp, &rpp_info);
613
614 if (unlikely(rpp_info.txch >= RTW89_TXCH_NUM ||
615 info->tx_dma_ch_mask & BIT(rpp_info.txch))) {
616 rtw89_warn(rtwdev, "should no release report on txch %d\n",
617 rpp_info.txch);
618 return;
619 }
620
621 if (unlikely(rpp_info.seq >= RTW89_PCI_TXWD_NUM_MAX)) {
622 rtw89_warn(rtwdev, "invalid seq %d\n", rpp_info.seq);
623 return;
624 }
625
626 tx_ring = &rtwpci->tx.rings[rpp_info.txch];
627 wd_ring = &tx_ring->wd_ring;
628 txwd = &wd_ring->pages[rpp_info.seq];
629
630 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, rpp_info.seq,
631 rpp_info.tx_status);
632 }
633
rtw89_pci_release_pending_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)634 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
635 struct rtw89_pci_tx_ring *tx_ring)
636 {
637 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
638 struct rtw89_pci_tx_wd *txwd;
639 int i;
640
641 for (i = 0; i < wd_ring->page_num; i++) {
642 txwd = &wd_ring->pages[i];
643
644 if (!list_empty(&txwd->list))
645 continue;
646
647 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
648 }
649 }
650
rtw89_pci_release_tx_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 max_cnt)651 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
652 struct rtw89_pci_rx_ring *rx_ring,
653 u32 max_cnt)
654 {
655 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
656 const struct rtw89_pci_info *info = rtwdev->pci_info;
657 struct rtw89_rx_desc_info desc_info = {};
658 struct rtw89_pci_rx_info *rx_info;
659 struct sk_buff *skb;
660 void *rpp;
661 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
662 u32 rpp_size = info->rpp_fmt_size;
663 u32 cnt = 0;
664 u32 skb_idx;
665 u32 offset;
666 int ret;
667
668 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
669 skb = rx_ring->buf[skb_idx];
670
671 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
672 if (ret) {
673 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
674 bd_ring->wp, ret);
675 goto err_sync_device;
676 }
677
678 rx_info = RTW89_PCI_RX_SKB_CB(skb);
679 if (!rx_info->fs || !rx_info->ls) {
680 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
681 return cnt;
682 }
683
684 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
685
686 /* first segment has RX desc */
687 offset = desc_info.offset + desc_info.rxd_len;
688 for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
689 rpp = skb->data + offset;
690 rtw89_pci_release_rpp(rtwdev, rpp);
691 }
692
693 rtw89_pci_sync_skb_for_device(rtwdev, skb);
694 rtw89_pci_rxbd_increase(rx_ring, 1);
695 cnt++;
696
697 return cnt;
698
699 err_sync_device:
700 rtw89_pci_sync_skb_for_device(rtwdev, skb);
701 return 0;
702 }
703
rtw89_pci_release_tx(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)704 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
705 struct rtw89_pci_rx_ring *rx_ring,
706 u32 cnt)
707 {
708 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
709 u32 release_cnt;
710
711 while (cnt) {
712 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
713 if (!release_cnt) {
714 rtw89_err(rtwdev, "failed to release TX skbs\n");
715
716 /* skip the rest RXBD bufs */
717 rtw89_pci_rxbd_increase(rx_ring, cnt);
718 break;
719 }
720
721 cnt -= release_cnt;
722 }
723
724 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
725 }
726
rtw89_pci_poll_rpq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)727 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
728 struct rtw89_pci *rtwpci, int budget)
729 {
730 struct rtw89_pci_rx_ring *rx_ring;
731 u32 cnt;
732 int work_done;
733
734 rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RPQ];
735
736 spin_lock_bh(&rtwpci->trx_lock);
737
738 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
739 if (cnt == 0)
740 goto out_unlock;
741
742 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
743
744 out_unlock:
745 spin_unlock_bh(&rtwpci->trx_lock);
746
747 /* always release all RPQ */
748 work_done = min_t(int, cnt, budget);
749 rtwdev->napi_budget_countdown -= work_done;
750
751 return work_done;
752 }
753
rtw89_pci_isr_rxd_unavail(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)754 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
755 struct rtw89_pci *rtwpci)
756 {
757 struct rtw89_pci_rx_ring *rx_ring;
758 struct rtw89_pci_dma_ring *bd_ring;
759 u32 reg_idx;
760 u16 hw_idx, hw_idx_next, host_idx;
761 int i;
762
763 for (i = 0; i < RTW89_RXCH_NUM; i++) {
764 rx_ring = &rtwpci->rx.rings[i];
765 bd_ring = &rx_ring->bd_ring;
766
767 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
768 hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
769 host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
770 hw_idx_next = (hw_idx + 1) % bd_ring->len;
771
772 if (hw_idx_next == host_idx)
773 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
774
775 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
776 "%d RXD unavailable, idx=0x%08x, len=%d\n",
777 i, reg_idx, bd_ring->len);
778 }
779 }
780
rtw89_pci_recognize_intrs(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)781 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
782 struct rtw89_pci *rtwpci,
783 struct rtw89_pci_isrs *isrs)
784 {
785 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
786 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
787 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
788
789 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
790 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
791 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
792 }
793 EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
794
rtw89_pci_recognize_intrs_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)795 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
796 struct rtw89_pci *rtwpci,
797 struct rtw89_pci_isrs *isrs)
798 {
799 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
800 isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
801 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
802 isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
803 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
804 isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
805 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
806
807 if (isrs->halt_c2h_isrs)
808 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
809 if (isrs->isrs[0])
810 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
811 if (isrs->isrs[1])
812 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
813 }
814 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
815
rtw89_pci_recognize_intrs_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)816 void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
817 struct rtw89_pci *rtwpci,
818 struct rtw89_pci_isrs *isrs)
819 {
820 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
821 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ?
822 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
823 isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ?
824 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
825 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1];
826
827 if (isrs->halt_c2h_isrs)
828 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
829 if (isrs->isrs[0])
830 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]);
831 if (isrs->isrs[1])
832 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
833 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
834 }
835 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2);
836
rtw89_pci_recognize_intrs_v3(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)837 void rtw89_pci_recognize_intrs_v3(struct rtw89_dev *rtwdev,
838 struct rtw89_pci *rtwpci,
839 struct rtw89_pci_isrs *isrs)
840 {
841 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
842 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ?
843 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
844 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1];
845
846 /* isrs[0] is not used, so borrow to store RDU status to share common
847 * flow in rtw89_pci_interrupt_threadfn().
848 */
849 isrs->isrs[0] = isrs->isrs[1] & (B_BE_PCIE_RDU_CH1_INT |
850 B_BE_PCIE_RDU_CH0_INT);
851
852 if (isrs->halt_c2h_isrs)
853 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
854 if (isrs->isrs[1])
855 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
856 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
857 }
858 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v3);
859
rtw89_pci_enable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)860 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
861 {
862 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
863 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
864 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
865 }
866 EXPORT_SYMBOL(rtw89_pci_enable_intr);
867
rtw89_pci_disable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)868 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
869 {
870 rtw89_write32(rtwdev, R_AX_HIMR0, 0);
871 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
872 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
873 }
874 EXPORT_SYMBOL(rtw89_pci_disable_intr);
875
rtw89_pci_enable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)876 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
877 {
878 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
879 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
880 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
881 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
882 }
883 EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
884
rtw89_pci_disable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)885 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
886 {
887 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
888 }
889 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
890
rtw89_pci_enable_intr_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)891 void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
892 {
893 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
894 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]);
895 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
896 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
897 }
898 EXPORT_SYMBOL(rtw89_pci_enable_intr_v2);
899
rtw89_pci_disable_intr_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)900 void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
901 {
902 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
903 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
904 }
905 EXPORT_SYMBOL(rtw89_pci_disable_intr_v2);
906
rtw89_pci_enable_intr_v3(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)907 void rtw89_pci_enable_intr_v3(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
908 {
909 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
910 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
911 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
912 }
913 EXPORT_SYMBOL(rtw89_pci_enable_intr_v3);
914
rtw89_pci_disable_intr_v3(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)915 void rtw89_pci_disable_intr_v3(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
916 {
917 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
918 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
919 }
920 EXPORT_SYMBOL(rtw89_pci_disable_intr_v3);
921
rtw89_pci_ops_recovery_start(struct rtw89_dev * rtwdev)922 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
923 {
924 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
925 unsigned long flags;
926
927 spin_lock_irqsave(&rtwpci->irq_lock, flags);
928 rtw89_chip_disable_intr(rtwdev, rtwpci);
929 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
930 rtw89_chip_enable_intr(rtwdev, rtwpci);
931 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
932 }
933
rtw89_pci_ops_recovery_complete(struct rtw89_dev * rtwdev)934 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
935 {
936 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
937 unsigned long flags;
938
939 spin_lock_irqsave(&rtwpci->irq_lock, flags);
940 rtw89_chip_disable_intr(rtwdev, rtwpci);
941 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
942 rtw89_chip_enable_intr(rtwdev, rtwpci);
943 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
944 }
945
rtw89_pci_low_power_interrupt_handler(struct rtw89_dev * rtwdev)946 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
947 {
948 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
949 int budget = NAPI_POLL_WEIGHT;
950
951 /* To prevent RXQ get stuck due to run out of budget. */
952 rtwdev->napi_budget_countdown = budget;
953
954 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
955 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
956 }
957
rtw89_pci_interrupt_threadfn(int irq,void * dev)958 static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
959 {
960 struct rtw89_dev *rtwdev = dev;
961 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
962 const struct rtw89_pci_info *info = rtwdev->pci_info;
963 const struct rtw89_pci_isr_def *isr_def = info->isr_def;
964 struct rtw89_pci_isrs isrs;
965 unsigned long flags;
966
967 spin_lock_irqsave(&rtwpci->irq_lock, flags);
968 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
969 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
970
971 if (unlikely(isrs.isrs[0] & isr_def->isr_rdu))
972 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
973
974 if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_halt_c2h))
975 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
976
977 if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_wdt_timeout))
978 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
979
980 if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_sps_ocp))
981 rtw89_warn(rtwdev, "SPS OCP alarm 0x%x\n", isrs.halt_c2h_isrs);
982
983 if (unlikely(rtwpci->under_recovery))
984 goto enable_intr;
985
986 if (unlikely(rtwpci->low_power)) {
987 rtw89_pci_low_power_interrupt_handler(rtwdev);
988 goto enable_intr;
989 }
990
991 if (likely(rtwpci->running)) {
992 local_bh_disable();
993 napi_schedule(&rtwdev->napi);
994 local_bh_enable();
995 }
996
997 return IRQ_HANDLED;
998
999 enable_intr:
1000 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1001 if (likely(rtwpci->running))
1002 rtw89_chip_enable_intr(rtwdev, rtwpci);
1003 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1004 return IRQ_HANDLED;
1005 }
1006
rtw89_pci_interrupt_handler(int irq,void * dev)1007 static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
1008 {
1009 struct rtw89_dev *rtwdev = dev;
1010 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1011 unsigned long flags;
1012 irqreturn_t irqret = IRQ_WAKE_THREAD;
1013
1014 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1015
1016 /* If interrupt event is on the road, it is still trigger interrupt
1017 * even we have done pci_stop() to turn off IMR.
1018 */
1019 if (unlikely(!rtwpci->running)) {
1020 irqret = IRQ_HANDLED;
1021 goto exit;
1022 }
1023
1024 rtw89_chip_disable_intr(rtwdev, rtwpci);
1025 exit:
1026 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1027
1028 return irqret;
1029 }
1030
1031 #define DEF_TXCHADDRS_TYPE3(gen, ch_idx, txch, v...) \
1032 [RTW89_TXCH_##ch_idx] = { \
1033 .num = R_##gen##_##txch##_TXBD_CFG, \
1034 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \
1035 .bdram = 0, \
1036 .desa_l = 0, \
1037 .desa_h = 0, \
1038 }
1039
1040 #define DEF_TXCHADDRS_TYPE3_GRP_BASE(gen, ch_idx, txch, grp, v...) \
1041 [RTW89_TXCH_##ch_idx] = { \
1042 .num = R_##gen##_##txch##_TXBD_CFG, \
1043 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \
1044 .bdram = 0, \
1045 .desa_l = R_##gen##_##grp##_TXBD_DESA_L, \
1046 .desa_h = R_##gen##_##grp##_TXBD_DESA_H, \
1047 }
1048
1049 #define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \
1050 [RTW89_TXCH_##ch_idx] = { \
1051 .num = R_##gen##_##txch##_TXBD_NUM ##v, \
1052 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \
1053 .bdram = 0, \
1054 .desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \
1055 .desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \
1056 }
1057
1058 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
1059 [RTW89_TXCH_##txch] = { \
1060 .num = R_AX_##txch##_TXBD_NUM ##v, \
1061 .idx = R_AX_##txch##_TXBD_IDX ##v, \
1062 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
1063 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
1064 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
1065 }
1066
1067 #define DEF_TXCHADDRS(info, txch, v...) \
1068 [RTW89_TXCH_##txch] = { \
1069 .num = R_AX_##txch##_TXBD_NUM, \
1070 .idx = R_AX_##txch##_TXBD_IDX, \
1071 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
1072 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
1073 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
1074 }
1075
1076 #define DEF_RXCHADDRS_TYPE3(gen, ch_idx, rxch, v...) \
1077 [RTW89_RXCH_##ch_idx] = { \
1078 .num = R_##gen##_RX_##rxch##_RXBD_CONFIG, \
1079 .idx = R_##gen##_##ch_idx##0_RXBD_IDX ##v, \
1080 .desa_l = 0, \
1081 .desa_h = 0, \
1082 }
1083
1084 #define DEF_RXCHADDRS_TYPE3_GRP_BASE(gen, ch_idx, rxch, grp, v...) \
1085 [RTW89_RXCH_##ch_idx] = { \
1086 .num = R_##gen##_RX_##rxch##_RXBD_CONFIG, \
1087 .idx = R_##gen##_##ch_idx##0_RXBD_IDX ##v, \
1088 .desa_l = R_##gen##_##grp##_RXBD_DESA_L, \
1089 .desa_h = R_##gen##_##grp##_RXBD_DESA_H, \
1090 }
1091
1092 #define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \
1093 [RTW89_RXCH_##ch_idx] = { \
1094 .num = R_##gen##_##rxch##_RXBD_NUM ##v, \
1095 .idx = R_##gen##_##rxch##_RXBD_IDX ##v, \
1096 .desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \
1097 .desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \
1098 }
1099
1100 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
1101 .tx = {
1102 DEF_TXCHADDRS(info, ACH0),
1103 DEF_TXCHADDRS(info, ACH1),
1104 DEF_TXCHADDRS(info, ACH2),
1105 DEF_TXCHADDRS(info, ACH3),
1106 DEF_TXCHADDRS(info, ACH4),
1107 DEF_TXCHADDRS(info, ACH5),
1108 DEF_TXCHADDRS(info, ACH6),
1109 DEF_TXCHADDRS(info, ACH7),
1110 DEF_TXCHADDRS(info, CH8),
1111 DEF_TXCHADDRS(info, CH9),
1112 DEF_TXCHADDRS_TYPE1(info, CH10),
1113 DEF_TXCHADDRS_TYPE1(info, CH11),
1114 DEF_TXCHADDRS(info, CH12),
1115 },
1116 .rx = {
1117 DEF_RXCHADDRS(AX, RXQ, RXQ),
1118 DEF_RXCHADDRS(AX, RPQ, RPQ),
1119 },
1120 };
1121 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
1122
1123 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
1124 .tx = {
1125 DEF_TXCHADDRS(info, ACH0, _V1),
1126 DEF_TXCHADDRS(info, ACH1, _V1),
1127 DEF_TXCHADDRS(info, ACH2, _V1),
1128 DEF_TXCHADDRS(info, ACH3, _V1),
1129 DEF_TXCHADDRS(info, ACH4, _V1),
1130 DEF_TXCHADDRS(info, ACH5, _V1),
1131 DEF_TXCHADDRS(info, ACH6, _V1),
1132 DEF_TXCHADDRS(info, ACH7, _V1),
1133 DEF_TXCHADDRS(info, CH8, _V1),
1134 DEF_TXCHADDRS(info, CH9, _V1),
1135 DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
1136 DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
1137 DEF_TXCHADDRS(info, CH12, _V1),
1138 },
1139 .rx = {
1140 DEF_RXCHADDRS(AX, RXQ, RXQ, _V1),
1141 DEF_RXCHADDRS(AX, RPQ, RPQ, _V1),
1142 },
1143 };
1144 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
1145
1146 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = {
1147 .tx = {
1148 DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1),
1149 DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1),
1150 DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1),
1151 DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1),
1152 DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1),
1153 DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1),
1154 DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1),
1155 DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1),
1156 DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1),
1157 DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1),
1158 DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1),
1159 DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1),
1160 DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1),
1161 },
1162 .rx = {
1163 DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1),
1164 DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1),
1165 },
1166 };
1167 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be);
1168
1169 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be_v1 = {
1170 .tx = {
1171 DEF_TXCHADDRS_TYPE3_GRP_BASE(BE, ACH0, CH0, ACQ, _V1),
1172 /* no CH1 */
1173 DEF_TXCHADDRS_TYPE3(BE, ACH2, CH2, _V1),
1174 /* no CH3 */
1175 DEF_TXCHADDRS_TYPE3(BE, ACH4, CH4, _V1),
1176 /* no CH5 */
1177 DEF_TXCHADDRS_TYPE3(BE, ACH6, CH6, _V1),
1178 /* no CH7 */
1179 DEF_TXCHADDRS_TYPE3_GRP_BASE(BE, CH8, CH8, NACQ, _V1),
1180 /* no CH9 */
1181 DEF_TXCHADDRS_TYPE3(BE, CH10, CH10, _V1),
1182 /* no CH11 */
1183 DEF_TXCHADDRS_TYPE3(BE, CH12, CH12, _V1),
1184 },
1185 .rx = {
1186 DEF_RXCHADDRS_TYPE3_GRP_BASE(BE, RXQ, CH0, HOST0, _V1),
1187 DEF_RXCHADDRS_TYPE3(BE, RPQ, CH1, _V1),
1188 },
1189 };
1190 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be_v1);
1191
1192 #undef DEF_TXCHADDRS_TYPE3
1193 #undef DEF_TXCHADDRS_TYPE3_GRP_BASE
1194 #undef DEF_TXCHADDRS_TYPE2
1195 #undef DEF_TXCHADDRS_TYPE1
1196 #undef DEF_TXCHADDRS
1197 #undef DEF_RXCHADDRS_TYPE3
1198 #undef DEF_RXCHADDRS_TYPE3_GRP_BASE
1199 #undef DEF_RXCHADDRS
1200
rtw89_pci_get_txch_addrs(struct rtw89_dev * rtwdev,enum rtw89_tx_channel txch,const struct rtw89_pci_ch_dma_addr ** addr)1201 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
1202 enum rtw89_tx_channel txch,
1203 const struct rtw89_pci_ch_dma_addr **addr)
1204 {
1205 const struct rtw89_pci_info *info = rtwdev->pci_info;
1206
1207 if (txch >= RTW89_TXCH_NUM)
1208 return -EINVAL;
1209
1210 *addr = &info->dma_addr_set->tx[txch];
1211
1212 return 0;
1213 }
1214
rtw89_pci_get_rxch_addrs(struct rtw89_dev * rtwdev,enum rtw89_rx_channel rxch,const struct rtw89_pci_ch_dma_addr ** addr)1215 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
1216 enum rtw89_rx_channel rxch,
1217 const struct rtw89_pci_ch_dma_addr **addr)
1218 {
1219 const struct rtw89_pci_info *info = rtwdev->pci_info;
1220
1221 if (rxch >= RTW89_RXCH_NUM)
1222 return -EINVAL;
1223
1224 *addr = &info->dma_addr_set->rx[rxch];
1225
1226 return 0;
1227 }
1228
rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring * ring)1229 static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
1230 {
1231 struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
1232
1233 /* reserved 1 desc check ring is full or not */
1234 if (bd_ring->rp > bd_ring->wp)
1235 return bd_ring->rp - bd_ring->wp - 1;
1236
1237 return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
1238 }
1239
1240 static
__rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev * rtwdev)1241 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
1242 {
1243 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1244 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[RTW89_TXCH_CH12];
1245 u32 cnt;
1246
1247 spin_lock_bh(&rtwpci->trx_lock);
1248 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
1249 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1250 spin_unlock_bh(&rtwpci->trx_lock);
1251
1252 return cnt;
1253 }
1254
1255 static
__rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev * rtwdev,u8 txch)1256 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
1257 u8 txch)
1258 {
1259 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1260 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch];
1261 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1262 u32 cnt;
1263
1264 spin_lock_bh(&rtwpci->trx_lock);
1265 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1266 if (txch != RTW89_TXCH_CH12)
1267 cnt = min(cnt, wd_ring->curr_num);
1268 spin_unlock_bh(&rtwpci->trx_lock);
1269
1270 return cnt;
1271 }
1272
__rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)1273 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1274 u8 txch)
1275 {
1276 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1277 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch];
1278 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1279 const struct rtw89_chip_info *chip = rtwdev->chip;
1280 u32 bd_cnt, wd_cnt, min_cnt = 0;
1281 struct rtw89_pci_rx_ring *rx_ring;
1282 enum rtw89_debug_mask debug_mask;
1283 u32 cnt;
1284
1285 rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RPQ];
1286
1287 spin_lock_bh(&rtwpci->trx_lock);
1288 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1289 wd_cnt = wd_ring->curr_num;
1290
1291 if (wd_cnt == 0 || bd_cnt == 0) {
1292 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
1293 if (cnt)
1294 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
1295 else if (wd_cnt == 0)
1296 goto out_unlock;
1297
1298 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1299 if (bd_cnt == 0)
1300 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
1301 }
1302
1303 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1304 wd_cnt = wd_ring->curr_num;
1305 min_cnt = min(bd_cnt, wd_cnt);
1306 if (min_cnt == 0) {
1307 /* This message can be frequently shown in low power mode or
1308 * high traffic with small FIFO chips, and we have recognized it as normal
1309 * behavior, so print with mask RTW89_DBG_TXRX in these situations.
1310 */
1311 if (rtwpci->low_power || chip->small_fifo_size)
1312 debug_mask = RTW89_DBG_TXRX;
1313 else
1314 debug_mask = RTW89_DBG_UNEXP;
1315
1316 rtw89_debug(rtwdev, debug_mask,
1317 "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
1318 wd_cnt, bd_cnt);
1319 }
1320
1321 out_unlock:
1322 spin_unlock_bh(&rtwpci->trx_lock);
1323
1324 return min_cnt;
1325 }
1326
rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)1327 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1328 u8 txch)
1329 {
1330 if (rtwdev->hci.paused)
1331 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1332
1333 if (txch == RTW89_TXCH_CH12)
1334 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1335
1336 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1337 }
1338
__rtw89_pci_tx_kick_off(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1339 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1340 {
1341 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1342 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1343 u32 host_idx, addr;
1344
1345 spin_lock_bh(&rtwpci->trx_lock);
1346
1347 addr = bd_ring->addr.idx;
1348 host_idx = bd_ring->wp;
1349 rtw89_write16(rtwdev, addr, host_idx);
1350
1351 spin_unlock_bh(&rtwpci->trx_lock);
1352 }
1353
rtw89_pci_tx_bd_ring_update(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,int n_txbd)1354 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1355 int n_txbd)
1356 {
1357 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1358 u32 host_idx, len;
1359
1360 len = bd_ring->len;
1361 host_idx = bd_ring->wp + n_txbd;
1362 host_idx = host_idx < len ? host_idx : host_idx - len;
1363
1364 bd_ring->wp = host_idx;
1365 }
1366
rtw89_pci_ops_tx_kick_off(struct rtw89_dev * rtwdev,u8 txch)1367 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1368 {
1369 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1370 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch];
1371
1372 if (rtwdev->hci.paused) {
1373 set_bit(txch, rtwpci->kick_map);
1374 return;
1375 }
1376
1377 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1378 }
1379
rtw89_pci_tx_kick_off_pending(struct rtw89_dev * rtwdev)1380 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1381 {
1382 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1383 struct rtw89_pci_tx_ring *tx_ring;
1384 int txch;
1385
1386 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1387 if (!test_and_clear_bit(txch, rtwpci->kick_map))
1388 continue;
1389
1390 tx_ring = &rtwpci->tx.rings[txch];
1391 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1392 }
1393 }
1394
__pci_flush_txch(struct rtw89_dev * rtwdev,u8 txch,bool drop)1395 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1396 {
1397 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1398 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch];
1399 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1400 u32 cur_idx, cur_rp;
1401 u8 i;
1402
1403 /* Because the time taked by the I/O is a bit dynamic, it's hard to
1404 * define a reasonable fixed total timeout to use read_poll_timeout*
1405 * helper. Instead, we can ensure a reasonable polling times, so we
1406 * just use for loop with udelay here.
1407 */
1408 for (i = 0; i < 60; i++) {
1409 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1410 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1411 if (cur_rp == bd_ring->wp)
1412 return;
1413
1414 udelay(1);
1415 }
1416
1417 if (!drop)
1418 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1419 }
1420
__rtw89_pci_ops_flush_txchs(struct rtw89_dev * rtwdev,u32 txchs,bool drop)1421 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1422 bool drop)
1423 {
1424 const struct rtw89_pci_info *info = rtwdev->pci_info;
1425 u8 i;
1426
1427 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1428 /* It may be unnecessary to flush FWCMD queue. */
1429 if (i == RTW89_TXCH_CH12)
1430 continue;
1431 if (info->tx_dma_ch_mask & BIT(i))
1432 continue;
1433
1434 if (txchs & BIT(i))
1435 __pci_flush_txch(rtwdev, i, drop);
1436 }
1437 }
1438
rtw89_pci_ops_flush_queues(struct rtw89_dev * rtwdev,u32 queues,bool drop)1439 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1440 bool drop)
1441 {
1442 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1443 }
1444
rtw89_pci_fill_txaddr_info(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1445 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1446 void *txaddr_info_addr, u32 total_len,
1447 dma_addr_t dma, u8 *add_info_nr)
1448 {
1449 struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1450 __le16 option;
1451
1452 txaddr_info->length = cpu_to_le16(total_len);
1453 option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS | RTW89_PCI_ADDR_NUM(1));
1454 option |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_ADDR_HIGH_MASK);
1455 txaddr_info->option = option;
1456 txaddr_info->dma = cpu_to_le32(dma);
1457
1458 *add_info_nr = 1;
1459
1460 return sizeof(*txaddr_info);
1461 }
1462 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1463
rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1464 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1465 void *txaddr_info_addr, u32 total_len,
1466 dma_addr_t dma, u8 *add_info_nr)
1467 {
1468 struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1469 u32 remain = total_len;
1470 u32 len;
1471 u16 length_option;
1472 int n;
1473
1474 for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1475 len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1476 TXADDR_INFO_LENTHG_V1_MAX : remain;
1477 remain -= len;
1478
1479 length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1480 FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1481 FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1482 length_option |= u16_encode_bits(upper_32_bits(dma),
1483 B_PCIADDR_HIGH_SEL_V1_MASK);
1484 txaddr_info->length_opt = cpu_to_le16(length_option);
1485 txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1486 txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1487
1488 dma += len;
1489 txaddr_info++;
1490 }
1491
1492 WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1493 remain, total_len);
1494
1495 *add_info_nr = n;
1496
1497 return n * sizeof(*txaddr_info);
1498 }
1499 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1500
rtw89_pci_txwd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,struct rtw89_core_tx_request * tx_req)1501 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1502 struct rtw89_pci_tx_ring *tx_ring,
1503 struct rtw89_pci_tx_wd *txwd,
1504 struct rtw89_core_tx_request *tx_req)
1505 {
1506 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1507 const struct rtw89_chip_info *chip = rtwdev->chip;
1508 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1509 struct rtw89_pci_tx_wp_info *txwp_info;
1510 void *txaddr_info_addr;
1511 struct pci_dev *pdev = rtwpci->pdev;
1512 struct sk_buff *skb = tx_req->skb;
1513 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1514 bool en_wd_info = desc_info->en_wd_info;
1515 u32 txwd_len;
1516 u32 txwp_len;
1517 u32 txaddr_info_len;
1518 dma_addr_t dma;
1519 int ret;
1520
1521 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1522 if (dma_mapping_error(&pdev->dev, dma)) {
1523 rtw89_err(rtwdev, "failed to map skb dma data\n");
1524 ret = -EBUSY;
1525 goto err;
1526 }
1527
1528 tx_data->dma = dma;
1529
1530 txwp_len = sizeof(*txwp_info);
1531 txwd_len = chip->txwd_body_size;
1532 txwd_len += en_wd_info ? chip->txwd_info_size : 0;
1533
1534 #if defined(__linux__)
1535 txwp_info = txwd->vaddr + txwd_len;
1536 #elif defined(__FreeBSD__)
1537 txwp_info = (struct rtw89_pci_tx_wp_info *)((u8 *)txwd->vaddr + txwd_len);
1538 #endif
1539 txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1540 txwp_info->seq1 = 0;
1541 txwp_info->seq2 = 0;
1542 txwp_info->seq3 = 0;
1543
1544 tx_ring->tx_cnt++;
1545 #if defined(__linux__)
1546 txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1547 #elif defined(__FreeBSD__)
1548 txaddr_info_addr = (u8 *)txwd->vaddr + txwd_len + txwp_len;
1549 #endif
1550 txaddr_info_len =
1551 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1552 dma, &desc_info->addr_info_nr);
1553
1554 txwd->len = txwd_len + txwp_len + txaddr_info_len;
1555
1556 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1557
1558 skb_queue_tail(&txwd->queue, skb);
1559
1560 return 0;
1561
1562 err:
1563 return ret;
1564 }
1565
rtw89_pci_fwcmd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1566 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1567 struct rtw89_pci_tx_ring *tx_ring,
1568 struct rtw89_pci_tx_bd_32 *txbd,
1569 struct rtw89_core_tx_request *tx_req)
1570 {
1571 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1572 const struct rtw89_chip_info *chip = rtwdev->chip;
1573 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1574 void *txdesc;
1575 int txdesc_size = chip->h2c_desc_size;
1576 struct pci_dev *pdev = rtwpci->pdev;
1577 struct sk_buff *skb = tx_req->skb;
1578 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1579 dma_addr_t dma;
1580 __le16 opt;
1581
1582 txdesc = skb_push(skb, txdesc_size);
1583 memset(txdesc, 0, txdesc_size);
1584 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1585
1586 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1587 if (dma_mapping_error(&pdev->dev, dma)) {
1588 rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1589 return -EBUSY;
1590 }
1591
1592 tx_data->dma = dma;
1593 opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS);
1594 opt |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_TXBD_OPT_DMA_HI);
1595 txbd->opt = opt;
1596 txbd->length = cpu_to_le16(skb->len);
1597 txbd->dma = cpu_to_le32(tx_data->dma);
1598 skb_queue_tail(&rtwpci->h2c_queue, skb);
1599
1600 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1601
1602 return 0;
1603 }
1604
rtw89_pci_txbd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1605 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1606 struct rtw89_pci_tx_ring *tx_ring,
1607 struct rtw89_pci_tx_bd_32 *txbd,
1608 struct rtw89_core_tx_request *tx_req)
1609 {
1610 struct rtw89_pci_tx_wd *txwd;
1611 __le16 opt;
1612 int ret;
1613
1614 /* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1615 * buffer with WD BODY only. So here we don't need to check the free
1616 * pages of the wd ring.
1617 */
1618 if (tx_ring->txch == RTW89_TXCH_CH12)
1619 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1620
1621 txwd = rtw89_pci_dequeue_txwd(tx_ring);
1622 if (!txwd) {
1623 rtw89_err(rtwdev, "no available TXWD\n");
1624 ret = -ENOSPC;
1625 goto err;
1626 }
1627
1628 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1629 if (ret) {
1630 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1631 goto err_enqueue_wd;
1632 }
1633
1634 list_add_tail(&txwd->list, &tx_ring->busy_pages);
1635
1636 opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS);
1637 opt |= le16_encode_bits(upper_32_bits(txwd->paddr), RTW89_PCI_TXBD_OPT_DMA_HI);
1638 txbd->opt = opt;
1639 txbd->length = cpu_to_le16(txwd->len);
1640 txbd->dma = cpu_to_le32(txwd->paddr);
1641
1642 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1643
1644 return 0;
1645
1646 err_enqueue_wd:
1647 rtw89_pci_enqueue_txwd(tx_ring, txwd);
1648 err:
1649 return ret;
1650 }
1651
rtw89_pci_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req,u8 txch)1652 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1653 u8 txch)
1654 {
1655 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1656 struct rtw89_pci_tx_ring *tx_ring;
1657 struct rtw89_pci_tx_bd_32 *txbd;
1658 u32 n_avail_txbd;
1659 int ret = 0;
1660
1661 /* check the tx type and dma channel for fw cmd queue */
1662 if ((txch == RTW89_TXCH_CH12 ||
1663 tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1664 (txch != RTW89_TXCH_CH12 ||
1665 tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1666 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1667 return -EINVAL;
1668 }
1669
1670 tx_ring = &rtwpci->tx.rings[txch];
1671 spin_lock_bh(&rtwpci->trx_lock);
1672
1673 n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring);
1674 if (n_avail_txbd == 0) {
1675 rtw89_err(rtwdev, "no available TXBD\n");
1676 ret = -ENOSPC;
1677 goto err_unlock;
1678 }
1679
1680 txbd = rtw89_pci_get_next_txbd(tx_ring);
1681 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1682 if (ret) {
1683 rtw89_err(rtwdev, "failed to submit TXBD\n");
1684 goto err_unlock;
1685 }
1686
1687 spin_unlock_bh(&rtwpci->trx_lock);
1688 return 0;
1689
1690 err_unlock:
1691 spin_unlock_bh(&rtwpci->trx_lock);
1692 return ret;
1693 }
1694
rtw89_pci_ops_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req)1695 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1696 {
1697 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1698 int ret;
1699
1700 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1701 if (ret) {
1702 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1703 return ret;
1704 }
1705
1706 return 0;
1707 }
1708
1709 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = {
1710 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1711 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1712 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1713 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1714 [RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1715 [RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1716 [RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1717 [RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1718 [RTW89_TXCH_CH8] = {.start_idx = 40, .max_num = 5, .min_num = 1},
1719 [RTW89_TXCH_CH9] = {.start_idx = 45, .max_num = 5, .min_num = 1},
1720 [RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1721 [RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1722 [RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1723 };
1724 EXPORT_SYMBOL(rtw89_bd_ram_table_dual);
1725
1726 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = {
1727 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1728 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1729 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1730 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1731 [RTW89_TXCH_CH8] = {.start_idx = 20, .max_num = 4, .min_num = 1},
1732 [RTW89_TXCH_CH9] = {.start_idx = 24, .max_num = 4, .min_num = 1},
1733 [RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1},
1734 };
1735 EXPORT_SYMBOL(rtw89_bd_ram_table_single);
1736
rtw89_pci_init_wp_16sel(struct rtw89_dev * rtwdev)1737 static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev)
1738 {
1739 const struct rtw89_pci_info *info = rtwdev->pci_info;
1740 u32 addr = info->wp_sel_addr;
1741 u32 val;
1742 int i;
1743
1744 if (!info->wp_sel_addr)
1745 return;
1746
1747 for (i = 0; i < 16; i += 4) {
1748 val = u32_encode_bits(i + 0, MASKBYTE0) |
1749 u32_encode_bits(i + 1, MASKBYTE1) |
1750 u32_encode_bits(i + 2, MASKBYTE2) |
1751 u32_encode_bits(i + 3, MASKBYTE3);
1752 rtw89_write32(rtwdev, addr + i, val);
1753 }
1754 }
1755
rtw89_pci_enc_bd_cfg(struct rtw89_dev * rtwdev,u16 bd_num,u32 dma_offset)1756 static u16 rtw89_pci_enc_bd_cfg(struct rtw89_dev *rtwdev, u16 bd_num,
1757 u32 dma_offset)
1758 {
1759 u16 dma_offset_sel;
1760 u16 num_sel;
1761
1762 /* B_BE_TX_NUM_SEL_MASK, B_BE_RX_NUM_SEL_MASK:
1763 * 0 -> 0
1764 * 1 -> 64 = 2^6
1765 * 2 -> 128 = 2^7
1766 * ...
1767 * 7 -> 4096 = 2^12
1768 */
1769 num_sel = ilog2(bd_num) - 5;
1770
1771 if (hweight16(bd_num) != 1)
1772 rtw89_warn(rtwdev, "bd_num %u is not power of 2\n", bd_num);
1773
1774 /* B_BE_TX_START_OFFSET_MASK, B_BE_RX_START_OFFSET_MASK:
1775 * 0 -> 0 = 0 * 2^9
1776 * 1 -> 512 = 1 * 2^9
1777 * 2 -> 1024 = 2 * 2^9
1778 * 3 -> 1536 = 3 * 2^9
1779 * ...
1780 * 255 -> 130560 = 255 * 2^9
1781 */
1782 dma_offset_sel = dma_offset >> 9;
1783
1784 if (dma_offset % 512)
1785 rtw89_warn(rtwdev, "offset %u is not multiple of 512\n", dma_offset);
1786
1787 return u16_encode_bits(num_sel, B_BE_TX_NUM_SEL_MASK) |
1788 u16_encode_bits(dma_offset_sel, B_BE_TX_START_OFFSET_MASK);
1789 }
1790
rtw89_pci_reset_trx_rings(struct rtw89_dev * rtwdev)1791 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1792 {
1793 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1794 const struct rtw89_pci_info *info = rtwdev->pci_info;
1795 const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table;
1796 struct rtw89_pci_tx_ring *tx_ring;
1797 struct rtw89_pci_rx_ring *rx_ring;
1798 struct rtw89_pci_dma_ring *bd_ring;
1799 const struct rtw89_pci_bd_ram *bd_ram;
1800 dma_addr_t group_dma_base = 0;
1801 u16 num_or_offset;
1802 u32 addr_desa_l;
1803 u32 addr_bdram;
1804 u32 addr_num;
1805 u32 addr_idx;
1806 u32 val32;
1807 int i;
1808
1809 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1810 if (info->tx_dma_ch_mask & BIT(i))
1811 continue;
1812
1813 tx_ring = &rtwpci->tx.rings[i];
1814 bd_ring = &tx_ring->bd_ring;
1815 bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL;
1816 addr_num = bd_ring->addr.num;
1817 addr_bdram = bd_ring->addr.bdram;
1818 addr_desa_l = bd_ring->addr.desa_l;
1819 bd_ring->wp = 0;
1820 bd_ring->rp = 0;
1821
1822 if (info->group_bd_addr) {
1823 if (addr_desa_l)
1824 group_dma_base = bd_ring->dma;
1825
1826 num_or_offset =
1827 rtw89_pci_enc_bd_cfg(rtwdev, bd_ring->len,
1828 bd_ring->dma - group_dma_base);
1829 } else {
1830 num_or_offset = bd_ring->len;
1831 }
1832 rtw89_write16(rtwdev, addr_num, num_or_offset);
1833
1834 if (addr_bdram && bd_ram) {
1835 val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1836 FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1837 FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1838
1839 rtw89_write32(rtwdev, addr_bdram, val32);
1840 }
1841 if (addr_desa_l) {
1842 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1843 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1844 }
1845 }
1846
1847 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1848 rx_ring = &rtwpci->rx.rings[i];
1849 bd_ring = &rx_ring->bd_ring;
1850 addr_num = bd_ring->addr.num;
1851 addr_idx = bd_ring->addr.idx;
1852 addr_desa_l = bd_ring->addr.desa_l;
1853 if (info->rx_ring_eq_is_full)
1854 bd_ring->wp = bd_ring->len - 1;
1855 else
1856 bd_ring->wp = 0;
1857 bd_ring->rp = 0;
1858 rx_ring->diliver_skb = NULL;
1859 rx_ring->diliver_desc.ready = false;
1860 rx_ring->target_rx_tag = 0;
1861
1862 if (info->group_bd_addr) {
1863 if (addr_desa_l)
1864 group_dma_base = bd_ring->dma;
1865
1866 num_or_offset =
1867 rtw89_pci_enc_bd_cfg(rtwdev, bd_ring->len,
1868 bd_ring->dma - group_dma_base);
1869 } else {
1870 num_or_offset = bd_ring->len;
1871 }
1872 rtw89_write16(rtwdev, addr_num, num_or_offset);
1873
1874 if (addr_desa_l) {
1875 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1876 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1877 }
1878
1879 if (info->rx_ring_eq_is_full)
1880 rtw89_write16(rtwdev, addr_idx, bd_ring->wp);
1881 }
1882
1883 rtw89_pci_init_wp_16sel(rtwdev);
1884 }
1885
rtw89_pci_release_tx_ring(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1886 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1887 struct rtw89_pci_tx_ring *tx_ring)
1888 {
1889 rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1890 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1891 }
1892
rtw89_pci_ops_reset(struct rtw89_dev * rtwdev)1893 void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1894 {
1895 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1896 const struct rtw89_pci_info *info = rtwdev->pci_info;
1897 int txch;
1898
1899 rtw89_pci_reset_trx_rings(rtwdev);
1900
1901 spin_lock_bh(&rtwpci->trx_lock);
1902 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1903 if (info->tx_dma_ch_mask & BIT(txch))
1904 continue;
1905 if (txch == RTW89_TXCH_CH12) {
1906 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1907 skb_queue_len(&rtwpci->h2c_queue), true);
1908 continue;
1909 }
1910 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx.rings[txch]);
1911 }
1912 spin_unlock_bh(&rtwpci->trx_lock);
1913 }
1914
rtw89_pci_enable_intr_lock(struct rtw89_dev * rtwdev)1915 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1916 {
1917 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1918 unsigned long flags;
1919
1920 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1921 rtwpci->running = true;
1922 rtw89_chip_enable_intr(rtwdev, rtwpci);
1923 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1924 }
1925
rtw89_pci_disable_intr_lock(struct rtw89_dev * rtwdev)1926 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1927 {
1928 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1929 unsigned long flags;
1930
1931 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1932 rtwpci->running = false;
1933 rtw89_chip_disable_intr(rtwdev, rtwpci);
1934 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1935 }
1936
rtw89_pci_ops_start(struct rtw89_dev * rtwdev)1937 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1938 {
1939 rtw89_core_napi_start(rtwdev);
1940 rtw89_pci_enable_intr_lock(rtwdev);
1941
1942 return 0;
1943 }
1944
rtw89_pci_ops_stop(struct rtw89_dev * rtwdev)1945 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1946 {
1947 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1948 struct pci_dev *pdev = rtwpci->pdev;
1949
1950 rtw89_pci_disable_intr_lock(rtwdev);
1951 synchronize_irq(pdev->irq);
1952 rtw89_core_napi_stop(rtwdev);
1953 }
1954
rtw89_pci_ops_pause(struct rtw89_dev * rtwdev,bool pause)1955 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1956 {
1957 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1958 struct pci_dev *pdev = rtwpci->pdev;
1959
1960 if (pause) {
1961 rtw89_pci_disable_intr_lock(rtwdev);
1962 synchronize_irq(pdev->irq);
1963 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1964 napi_synchronize(&rtwdev->napi);
1965 } else {
1966 rtw89_pci_enable_intr_lock(rtwdev);
1967 rtw89_pci_tx_kick_off_pending(rtwdev);
1968 }
1969 }
1970
1971 static
rtw89_pci_switch_bd_idx_addr(struct rtw89_dev * rtwdev,bool low_power)1972 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1973 {
1974 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1975 const struct rtw89_pci_info *info = rtwdev->pci_info;
1976 const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1977 const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1978 struct rtw89_pci_tx_ring *tx_ring;
1979 struct rtw89_pci_rx_ring *rx_ring;
1980 int i;
1981
1982 if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1983 return;
1984
1985 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1986 tx_ring = &rtwpci->tx.rings[i];
1987 tx_ring->bd_ring.addr.idx = low_power ?
1988 bd_idx_addr->tx_bd_addrs[i] :
1989 dma_addr_set->tx[i].idx;
1990 }
1991
1992 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1993 rx_ring = &rtwpci->rx.rings[i];
1994 rx_ring->bd_ring.addr.idx = low_power ?
1995 bd_idx_addr->rx_bd_addrs[i] :
1996 dma_addr_set->rx[i].idx;
1997 }
1998 }
1999
rtw89_pci_ops_switch_mode(struct rtw89_dev * rtwdev,bool low_power)2000 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
2001 {
2002 enum rtw89_pci_intr_mask_cfg cfg;
2003
2004 WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
2005
2006 cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
2007 rtw89_chip_config_intr_mask(rtwdev, cfg);
2008 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
2009 }
2010
2011 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
2012
rtw89_pci_ops_read32_cmac(struct rtw89_dev * rtwdev,u32 addr)2013 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
2014 {
2015 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2016 #if defined(__linux__)
2017 u32 val = readl(rtwpci->mmap + addr);
2018 #elif defined(__FreeBSD__)
2019 u32 val;
2020
2021 val = bus_read_4((struct resource *)rtwpci->mmap, addr);
2022 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
2023 #endif
2024 int count;
2025
2026 for (count = 0; ; count++) {
2027 if (val != RTW89_R32_DEAD)
2028 return val;
2029 if (count >= MAC_REG_POOL_COUNT) {
2030 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
2031 return RTW89_R32_DEAD;
2032 }
2033 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
2034 #if defined(__linux__)
2035 val = readl(rtwpci->mmap + addr);
2036 #elif defined(__FreeBSD__)
2037 val = bus_read_4((struct resource *)rtwpci->mmap, addr);
2038 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
2039 #endif
2040 }
2041
2042 return val;
2043 }
2044
rtw89_pci_ops_read8(struct rtw89_dev * rtwdev,u32 addr)2045 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
2046 {
2047 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2048 u32 addr32, val32, shift;
2049
2050 if (!ACCESS_CMAC(addr))
2051 #if defined(__linux__)
2052 return readb(rtwpci->mmap + addr);
2053 #elif defined(__FreeBSD__)
2054 {
2055 u8 val;
2056
2057 val = bus_read_1((struct resource *)rtwpci->mmap, addr);
2058 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val);
2059 return (val);
2060 }
2061 #endif
2062
2063 addr32 = addr & ~0x3;
2064 shift = (addr & 0x3) * 8;
2065 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
2066 return val32 >> shift;
2067 }
2068
rtw89_pci_ops_read16(struct rtw89_dev * rtwdev,u32 addr)2069 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
2070 {
2071 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2072 u32 addr32, val32, shift;
2073
2074 if (!ACCESS_CMAC(addr))
2075 #if defined(__linux__)
2076 return readw(rtwpci->mmap + addr);
2077 #elif defined(__FreeBSD__)
2078 {
2079 u16 val;
2080
2081 val = bus_read_2((struct resource *)rtwpci->mmap, addr);
2082 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val);
2083 return (val);
2084 }
2085 #endif
2086
2087 addr32 = addr & ~0x3;
2088 shift = (addr & 0x3) * 8;
2089 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
2090 return val32 >> shift;
2091 }
2092
rtw89_pci_ops_read32(struct rtw89_dev * rtwdev,u32 addr)2093 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
2094 {
2095 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2096
2097 if (!ACCESS_CMAC(addr))
2098 #if defined(__linux__)
2099 return readl(rtwpci->mmap + addr);
2100 #elif defined(__FreeBSD__)
2101 {
2102 u32 val;
2103
2104 val = bus_read_4((struct resource *)rtwpci->mmap, addr);
2105 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val);
2106 return (val);
2107 }
2108 #endif
2109
2110 return rtw89_pci_ops_read32_cmac(rtwdev, addr);
2111 }
2112
rtw89_pci_ops_write8(struct rtw89_dev * rtwdev,u32 addr,u8 data)2113 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
2114 {
2115 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2116
2117 #if defined(__linux__)
2118 writeb(data, rtwpci->mmap + addr);
2119 #elif defined(__FreeBSD__)
2120 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, data);
2121 return (bus_write_1((struct resource *)rtwpci->mmap, addr, data));
2122 #endif
2123 }
2124
rtw89_pci_ops_write16(struct rtw89_dev * rtwdev,u32 addr,u16 data)2125 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
2126 {
2127 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2128
2129 #if defined(__linux__)
2130 writew(data, rtwpci->mmap + addr);
2131 #elif defined(__FreeBSD__)
2132 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, data);
2133 return (bus_write_2((struct resource *)rtwpci->mmap, addr, data));
2134 #endif
2135 }
2136
rtw89_pci_ops_write32(struct rtw89_dev * rtwdev,u32 addr,u32 data)2137 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
2138 {
2139 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2140
2141 #if defined(__linux__)
2142 writel(data, rtwpci->mmap + addr);
2143 #elif defined(__FreeBSD__)
2144 rtw89_debug(rtwdev, RTW89_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, data);
2145 return (bus_write_4((struct resource *)rtwpci->mmap, addr, data));
2146 #endif
2147 }
2148
rtw89_pci_ops_read32_pci_cfg(struct rtw89_dev * rtwdev,u32 addr)2149 static u32 rtw89_pci_ops_read32_pci_cfg(struct rtw89_dev *rtwdev, u32 addr)
2150 {
2151 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2152 struct pci_dev *pdev = rtwpci->pdev;
2153 u32 value;
2154 int ret;
2155
2156 ret = pci_read_config_dword(pdev, addr, &value);
2157 if (ret)
2158 return RTW89_R32_EA;
2159
2160 return value;
2161 }
2162
rtw89_pci_ctrl_dma_trx(struct rtw89_dev * rtwdev,bool enable)2163 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
2164 {
2165 const struct rtw89_pci_info *info = rtwdev->pci_info;
2166
2167 if (enable)
2168 rtw89_write32_set(rtwdev, info->init_cfg_reg,
2169 info->rxhci_en_bit | info->txhci_en_bit);
2170 else
2171 rtw89_write32_clr(rtwdev, info->init_cfg_reg,
2172 info->rxhci_en_bit | info->txhci_en_bit);
2173 }
2174
rtw89_pci_ctrl_dma_io(struct rtw89_dev * rtwdev,bool enable)2175 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
2176 {
2177 const struct rtw89_pci_info *info = rtwdev->pci_info;
2178 const struct rtw89_reg_def *reg = &info->dma_io_stop;
2179
2180 if (enable)
2181 rtw89_write32_clr(rtwdev, reg->addr, reg->mask);
2182 else
2183 rtw89_write32_set(rtwdev, reg->addr, reg->mask);
2184 }
2185
rtw89_pci_ctrl_dma_all(struct rtw89_dev * rtwdev,bool enable)2186 void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
2187 {
2188 rtw89_pci_ctrl_dma_io(rtwdev, enable);
2189 rtw89_pci_ctrl_dma_trx(rtwdev, enable);
2190 }
2191
rtw89_pci_check_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 rw_bit)2192 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
2193 {
2194 u16 val;
2195
2196 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
2197
2198 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
2199 switch (speed) {
2200 case PCIE_PHY_GEN1:
2201 if (addr < 0x20)
2202 val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
2203 else
2204 val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
2205 break;
2206 case PCIE_PHY_GEN2:
2207 if (addr < 0x20)
2208 val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
2209 else
2210 val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
2211 break;
2212 default:
2213 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
2214 return -EINVAL;
2215 }
2216 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
2217 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
2218
2219 return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
2220 false, rtwdev, R_AX_MDIO_CFG);
2221 }
2222
2223 static int
rtw89_read16_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 * val)2224 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
2225 {
2226 int ret;
2227
2228 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
2229 if (ret) {
2230 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
2231 return ret;
2232 }
2233 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
2234
2235 return 0;
2236 }
2237
2238 static int
rtw89_write16_mdio(struct rtw89_dev * rtwdev,u8 addr,u16 data,u8 speed)2239 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
2240 {
2241 int ret;
2242
2243 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
2244 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
2245 if (ret) {
2246 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
2247 return ret;
2248 }
2249
2250 return 0;
2251 }
2252
2253 static int
rtw89_write16_mdio_mask(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u16 data,u8 speed)2254 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
2255 {
2256 u32 shift;
2257 int ret;
2258 u16 val;
2259
2260 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2261 if (ret)
2262 return ret;
2263
2264 shift = __ffs(mask);
2265 val &= ~mask;
2266 val |= ((data << shift) & mask);
2267
2268 ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
2269 if (ret)
2270 return ret;
2271
2272 return 0;
2273 }
2274
rtw89_write16_mdio_set(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)2275 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2276 {
2277 int ret;
2278 u16 val;
2279
2280 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2281 if (ret)
2282 return ret;
2283 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
2284 if (ret)
2285 return ret;
2286
2287 return 0;
2288 }
2289
rtw89_write16_mdio_clr(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)2290 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2291 {
2292 int ret;
2293 u16 val;
2294
2295 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
2296 if (ret)
2297 return ret;
2298 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
2299 if (ret)
2300 return ret;
2301
2302 return 0;
2303 }
2304
rtw89_dbi_write8(struct rtw89_dev * rtwdev,u16 addr,u8 data)2305 static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data)
2306 {
2307 u16 addr_2lsb = addr & B_AX_DBI_2LSB;
2308 u16 write_addr;
2309 u8 flag;
2310 int ret;
2311
2312 write_addr = addr & B_AX_DBI_ADDR_MSK;
2313 write_addr |= u16_encode_bits(BIT(addr_2lsb), B_AX_DBI_WREN_MSK);
2314 rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data);
2315 rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr);
2316 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16);
2317
2318 ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10,
2319 10 * RTW89_PCI_WR_RETRY_CNT, false,
2320 rtwdev, R_AX_DBI_FLAG + 2);
2321 if (ret)
2322 rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n",
2323 addr);
2324
2325 return ret;
2326 }
2327
rtw89_dbi_read8(struct rtw89_dev * rtwdev,u16 addr,u8 * value)2328 static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value)
2329 {
2330 u16 read_addr = addr & B_AX_DBI_ADDR_MSK;
2331 u8 flag;
2332 int ret;
2333
2334 rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr);
2335 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16);
2336
2337 ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10,
2338 10 * RTW89_PCI_WR_RETRY_CNT, false,
2339 rtwdev, R_AX_DBI_FLAG + 2);
2340 if (ret) {
2341 rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n",
2342 addr);
2343 return ret;
2344 }
2345
2346 read_addr = R_AX_DBI_RDATA + (addr & 3);
2347 *value = rtw89_read8(rtwdev, read_addr);
2348
2349 return 0;
2350 }
2351
rtw89_pci_write_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 data)2352 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2353 u8 data)
2354 {
2355 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2356 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2357 struct pci_dev *pdev = rtwpci->pdev;
2358 int ret;
2359
2360 ret = pci_write_config_byte(pdev, addr, data);
2361 if (!ret)
2362 return 0;
2363
2364 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2365 ret = rtw89_dbi_write8(rtwdev, addr, data);
2366
2367 return ret;
2368 }
2369
rtw89_pci_read_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 * value)2370 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2371 u8 *value)
2372 {
2373 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2374 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2375 struct pci_dev *pdev = rtwpci->pdev;
2376 int ret;
2377
2378 ret = pci_read_config_byte(pdev, addr, value);
2379 if (!ret)
2380 return 0;
2381
2382 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2383 ret = rtw89_dbi_read8(rtwdev, addr, value);
2384
2385 return ret;
2386 }
2387
rtw89_pci_config_byte_set(struct rtw89_dev * rtwdev,u16 addr,u8 bit)2388 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
2389 u8 bit)
2390 {
2391 u8 value;
2392 int ret;
2393
2394 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2395 if (ret)
2396 return ret;
2397
2398 value |= bit;
2399 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2400
2401 return ret;
2402 }
2403
rtw89_pci_config_byte_clr(struct rtw89_dev * rtwdev,u16 addr,u8 bit)2404 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
2405 u8 bit)
2406 {
2407 u8 value;
2408 int ret;
2409
2410 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
2411 if (ret)
2412 return ret;
2413
2414 value &= ~bit;
2415 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
2416
2417 return ret;
2418 }
2419
2420 static int
__get_target(struct rtw89_dev * rtwdev,u16 * target,enum rtw89_pcie_phy phy_rate)2421 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
2422 {
2423 u16 val, tar;
2424 int ret;
2425
2426 /* Enable counter */
2427 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
2428 if (ret)
2429 return ret;
2430 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2431 phy_rate);
2432 if (ret)
2433 return ret;
2434 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
2435 phy_rate);
2436 if (ret)
2437 return ret;
2438
2439 fsleep(300);
2440
2441 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
2442 if (ret)
2443 return ret;
2444 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
2445 phy_rate);
2446 if (ret)
2447 return ret;
2448
2449 tar = tar & 0x0FFF;
2450 if (tar == 0 || tar == 0x0FFF) {
2451 rtw89_err(rtwdev, "[ERR]Get target failed.\n");
2452 return -EINVAL;
2453 }
2454
2455 *target = tar;
2456
2457 return 0;
2458 }
2459
rtw89_pci_autok_x(struct rtw89_dev * rtwdev)2460 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
2461 {
2462 int ret;
2463
2464 if (!rtw89_is_rtl885xb(rtwdev))
2465 return 0;
2466
2467 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
2468 PCIE_AUTOK_4, PCIE_PHY_GEN1);
2469 return ret;
2470 }
2471
rtw89_pci_auto_refclk_cal(struct rtw89_dev * rtwdev,bool autook_en)2472 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
2473 {
2474 enum rtw89_pcie_phy phy_rate;
2475 u16 val16, mgn_set, div_set, tar;
2476 u8 val8, bdr_ori;
2477 bool l1_flag = false;
2478 int ret = 0;
2479
2480 if (!rtw89_is_rtl885xb(rtwdev))
2481 return 0;
2482
2483 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
2484 if (ret) {
2485 rtw89_err(rtwdev, "[ERR]pci config read %X\n",
2486 RTW89_PCIE_PHY_RATE);
2487 return ret;
2488 }
2489
2490 if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
2491 phy_rate = PCIE_PHY_GEN1;
2492 } else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
2493 phy_rate = PCIE_PHY_GEN2;
2494 } else {
2495 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
2496 return -EOPNOTSUPP;
2497 }
2498 /* Disable L1BD */
2499 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
2500 if (ret) {
2501 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
2502 return ret;
2503 }
2504
2505 if (bdr_ori & RTW89_PCIE_BIT_L1) {
2506 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2507 bdr_ori & ~RTW89_PCIE_BIT_L1);
2508 if (ret) {
2509 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2510 RTW89_PCIE_L1_CTRL);
2511 return ret;
2512 }
2513 l1_flag = true;
2514 }
2515
2516 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2517 if (ret) {
2518 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2519 goto end;
2520 }
2521
2522 if (val16 & B_AX_CALIB_EN) {
2523 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
2524 val16 & ~B_AX_CALIB_EN, phy_rate);
2525 if (ret) {
2526 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2527 goto end;
2528 }
2529 }
2530
2531 if (!autook_en)
2532 goto end;
2533 /* Set div */
2534 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
2535 if (ret) {
2536 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2537 goto end;
2538 }
2539
2540 /* Obtain div and margin */
2541 ret = __get_target(rtwdev, &tar, phy_rate);
2542 if (ret) {
2543 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2544 goto end;
2545 }
2546
2547 mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
2548
2549 if (mgn_set >= 128) {
2550 div_set = 0x0003;
2551 mgn_set = 0x000F;
2552 } else if (mgn_set >= 64) {
2553 div_set = 0x0003;
2554 mgn_set >>= 3;
2555 } else if (mgn_set >= 32) {
2556 div_set = 0x0002;
2557 mgn_set >>= 2;
2558 } else if (mgn_set >= 16) {
2559 div_set = 0x0001;
2560 mgn_set >>= 1;
2561 } else if (mgn_set == 0) {
2562 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2563 goto end;
2564 } else {
2565 div_set = 0x0000;
2566 }
2567
2568 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2569 if (ret) {
2570 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2571 goto end;
2572 }
2573
2574 val16 |= u16_encode_bits(div_set, B_AX_DIV);
2575
2576 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2577 if (ret) {
2578 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2579 goto end;
2580 }
2581
2582 ret = __get_target(rtwdev, &tar, phy_rate);
2583 if (ret) {
2584 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2585 goto end;
2586 }
2587
2588 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2589 tar, div_set, mgn_set);
2590 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2591 (tar & 0x0FFF) | (mgn_set << 12), phy_rate);
2592 if (ret) {
2593 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2594 goto end;
2595 }
2596
2597 /* Enable function */
2598 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2599 if (ret) {
2600 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2601 goto end;
2602 }
2603
2604 /* CLK delay = 0 */
2605 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2606 PCIE_CLKDLY_HW_0);
2607
2608 end:
2609 /* Set L1BD to ori */
2610 if (l1_flag) {
2611 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2612 bdr_ori);
2613 if (ret) {
2614 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2615 RTW89_PCIE_L1_CTRL);
2616 return ret;
2617 }
2618 }
2619
2620 return ret;
2621 }
2622
rtw89_pci_deglitch_setting(struct rtw89_dev * rtwdev)2623 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2624 {
2625 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2626 int ret;
2627
2628 if (chip_id == RTL8852A) {
2629 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2630 PCIE_PHY_GEN1);
2631 if (ret)
2632 return ret;
2633 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2634 PCIE_PHY_GEN2);
2635 if (ret)
2636 return ret;
2637 } else if (chip_id == RTL8852C) {
2638 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2639 B_AX_DEGLITCH);
2640 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2641 B_AX_DEGLITCH);
2642 }
2643
2644 return 0;
2645 }
2646
rtw89_pci_disable_eq_ax(struct rtw89_dev * rtwdev)2647 static void rtw89_pci_disable_eq_ax(struct rtw89_dev *rtwdev)
2648 {
2649 u16 g1_oobs, g2_oobs;
2650 u32 backup_aspm;
2651 u32 phy_offset;
2652 u16 offset_cal;
2653 u16 oobs_val;
2654 int ret;
2655 u8 gen;
2656
2657 if (rtwdev->chip->chip_id != RTL8852C)
2658 return;
2659
2660 g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2661 RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL);
2662 g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 +
2663 RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL);
2664 if (g1_oobs && g2_oobs)
2665 return;
2666
2667 backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1);
2668 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
2669
2670 ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, &phy_offset);
2671 if (ret)
2672 goto out;
2673
2674 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN);
2675 rtw89_write16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL);
2676 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL);
2677
2678 oobs_val = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT,
2679 OOBS_LEVEL_MASK);
2680
2681 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT,
2682 OOBS_SEN_MASK, oobs_val);
2683 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT,
2684 BAC_OOBS_SEL);
2685
2686 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT,
2687 OOBS_SEN_MASK, oobs_val);
2688 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT,
2689 BAC_OOBS_SEL);
2690
2691 /* offset K */
2692 for (gen = 1; gen <= 2; gen++) {
2693 phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 :
2694 R_RAC_DIRECT_OFFSET_G2;
2695
2696 rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
2697 B_PCIE_BIT_RD_SEL);
2698 }
2699
2700 offset_cal = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2701 RAC_ANA1F * RAC_MULT, OFFSET_CAL_MASK);
2702
2703 for (gen = 1; gen <= 2; gen++) {
2704 phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 :
2705 R_RAC_DIRECT_OFFSET_G2;
2706
2707 rtw89_write16_mask(rtwdev, phy_offset + RAC_ANA0B * RAC_MULT,
2708 MANUAL_LVL_MASK, offset_cal);
2709 rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT,
2710 OFFSET_CAL_MODE);
2711 }
2712
2713 out:
2714 rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, backup_aspm);
2715 }
2716
rtw89_pci_ber(struct rtw89_dev * rtwdev)2717 static void rtw89_pci_ber(struct rtw89_dev *rtwdev)
2718 {
2719 u32 phy_offset;
2720
2721 if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks))
2722 return;
2723
2724 phy_offset = R_RAC_DIRECT_OFFSET_G1;
2725 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL);
2726 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2727
2728 phy_offset = R_RAC_DIRECT_OFFSET_G2;
2729 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL);
2730 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2731 }
2732
rtw89_pci_rxdma_prefth(struct rtw89_dev * rtwdev)2733 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2734 {
2735 if (rtwdev->chip->chip_id != RTL8852A)
2736 return;
2737
2738 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2739 }
2740
rtw89_pci_l1off_pwroff(struct rtw89_dev * rtwdev)2741 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2742 {
2743 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2744
2745 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2746 return;
2747
2748 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2749 }
2750
rtw89_pci_l2_rxen_lat(struct rtw89_dev * rtwdev)2751 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2752 {
2753 int ret;
2754
2755 if (rtwdev->chip->chip_id != RTL8852A)
2756 return 0;
2757
2758 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2759 PCIE_PHY_GEN1);
2760 if (ret)
2761 return ret;
2762
2763 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2764 PCIE_PHY_GEN2);
2765 if (ret)
2766 return ret;
2767
2768 return 0;
2769 }
2770
rtw89_pci_aphy_pwrcut(struct rtw89_dev * rtwdev)2771 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2772 {
2773 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2774
2775 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2776 return;
2777
2778 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2779 }
2780
rtw89_pci_hci_ldo(struct rtw89_dev * rtwdev)2781 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2782 {
2783 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2784
2785 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2786 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2787 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2788 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2789 B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2790 } else if (rtwdev->chip->chip_id == RTL8852C) {
2791 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2792 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2793 }
2794 }
2795
rtw89_pci_dphy_delay(struct rtw89_dev * rtwdev)2796 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2797 {
2798 if (!rtw89_is_rtl885xb(rtwdev))
2799 return 0;
2800
2801 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2802 PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1);
2803 }
2804
rtw89_pci_power_wake_ax(struct rtw89_dev * rtwdev,bool pwr_up)2805 static void rtw89_pci_power_wake_ax(struct rtw89_dev *rtwdev, bool pwr_up)
2806 {
2807 if (pwr_up)
2808 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2809 else
2810 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2811 }
2812
rtw89_pci_autoload_hang(struct rtw89_dev * rtwdev)2813 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2814 {
2815 if (rtwdev->chip->chip_id != RTL8852C)
2816 return;
2817
2818 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2819 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2820 }
2821
rtw89_pci_l12_vmain(struct rtw89_dev * rtwdev)2822 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2823 {
2824 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2825 return;
2826
2827 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2828 }
2829
rtw89_pci_gen2_force_ib(struct rtw89_dev * rtwdev)2830 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2831 {
2832 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2833 return;
2834
2835 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2836 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2837 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2838 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2839 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2840 }
2841
rtw89_pci_l1_ent_lat(struct rtw89_dev * rtwdev)2842 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2843 {
2844 if (rtwdev->chip->chip_id != RTL8852C)
2845 return;
2846
2847 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2848 }
2849
rtw89_pci_wd_exit_l1(struct rtw89_dev * rtwdev)2850 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2851 {
2852 if (rtwdev->chip->chip_id != RTL8852C)
2853 return;
2854
2855 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2856 }
2857
rtw89_pci_set_sic(struct rtw89_dev * rtwdev)2858 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2859 {
2860 if (rtwdev->chip->chip_id == RTL8852C)
2861 return;
2862
2863 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2864 B_AX_SIC_EN_FORCE_CLKREQ);
2865 }
2866
rtw89_pci_set_lbc(struct rtw89_dev * rtwdev)2867 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2868 {
2869 const struct rtw89_pci_info *info = rtwdev->pci_info;
2870 u32 lbc;
2871
2872 if (rtwdev->chip->chip_id == RTL8852C)
2873 return;
2874
2875 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2876 if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2877 lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER);
2878 lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2879 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2880 } else {
2881 lbc &= ~B_AX_LBC_EN;
2882 }
2883 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2884 }
2885
rtw89_pci_set_io_rcy(struct rtw89_dev * rtwdev)2886 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2887 {
2888 const struct rtw89_pci_info *info = rtwdev->pci_info;
2889 u32 val32;
2890
2891 if (rtwdev->chip->chip_id != RTL8852C)
2892 return;
2893
2894 if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2895 val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2896 info->io_rcy_tmr);
2897 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2898 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2899 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2900
2901 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2902 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2903 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2904 } else {
2905 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2906 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2907 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2908 }
2909
2910 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2911 }
2912
rtw89_pci_set_dbg(struct rtw89_dev * rtwdev)2913 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2914 {
2915 if (rtwdev->chip->chip_id == RTL8852C)
2916 return;
2917
2918 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2919 B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2920
2921 rtw89_write32_mask(rtwdev, R_AX_PCIE_EXP_CTRL,
2922 B_AX_EN_STUCK_DBG | B_AX_ASFF_FULL_NO_STK,
2923 B_AX_EN_STUCK_DBG);
2924
2925 if (rtwdev->chip->chip_id == RTL8852A)
2926 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2927 B_AX_EN_CHKDSC_NO_RX_STUCK);
2928 }
2929
rtw89_pci_set_keep_reg(struct rtw89_dev * rtwdev)2930 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2931 {
2932 if (rtwdev->chip->chip_id == RTL8852C)
2933 return;
2934
2935 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2936 B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2937 }
2938
rtw89_pci_clr_idx_all_ax(struct rtw89_dev * rtwdev)2939 static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
2940 {
2941 const struct rtw89_pci_info *info = rtwdev->pci_info;
2942 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2943 u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2944 B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2945 B_AX_CLR_CH12_IDX;
2946 u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2947 u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2948
2949 if (chip_id == RTL8852A || chip_id == RTL8852C)
2950 val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2951 B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2952 /* clear DMA indexes */
2953 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2954 if (chip_id == RTL8852A || chip_id == RTL8852C)
2955 rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2956 B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2957 rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2958 B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2959 }
2960
rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev * rtwdev)2961 static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2962 {
2963 const struct rtw89_pci_info *info = rtwdev->pci_info;
2964 u32 dma_busy1 = info->dma_busy1.addr;
2965 u32 dma_busy2 = info->dma_busy2_reg;
2966 u32 check, dma_busy;
2967 int ret;
2968
2969 check = info->dma_busy1.mask;
2970
2971 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2972 10, 100, false, rtwdev, dma_busy1);
2973 if (ret)
2974 return ret;
2975
2976 if (!dma_busy2)
2977 return 0;
2978
2979 check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2980
2981 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2982 10, 100, false, rtwdev, dma_busy2);
2983 if (ret)
2984 return ret;
2985
2986 return 0;
2987 }
2988
rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev * rtwdev)2989 static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2990 {
2991 const struct rtw89_pci_info *info = rtwdev->pci_info;
2992 u32 dma_busy3 = info->dma_busy3_reg;
2993 u32 check, dma_busy;
2994 int ret;
2995
2996 check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2997
2998 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2999 10, 100, false, rtwdev, dma_busy3);
3000 if (ret)
3001 return ret;
3002
3003 return 0;
3004 }
3005
rtw89_pci_poll_dma_all_idle(struct rtw89_dev * rtwdev)3006 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
3007 {
3008 int ret;
3009
3010 ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev);
3011 if (ret) {
3012 rtw89_err(rtwdev, "txdma ch busy\n");
3013 return ret;
3014 }
3015
3016 ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev);
3017 if (ret) {
3018 rtw89_err(rtwdev, "rxdma ch busy\n");
3019 return ret;
3020 }
3021
3022 return 0;
3023 }
3024
rtw89_pci_mode_op(struct rtw89_dev * rtwdev)3025 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
3026 {
3027 const struct rtw89_pci_info *info = rtwdev->pci_info;
3028 enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
3029 enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
3030 enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
3031 enum mac_ax_tag_mode tag_mode = info->tag_mode;
3032 enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
3033 enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
3034 enum mac_ax_tx_burst tx_burst = info->tx_burst;
3035 enum mac_ax_rx_burst rx_burst = info->rx_burst;
3036 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3037 u8 cv = rtwdev->hal.cv;
3038 u32 val32;
3039
3040 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
3041 if (chip_id == RTL8852A && cv == CHIP_CBV)
3042 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
3043 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
3044 if (chip_id == RTL8852A || chip_id == RTL8852B)
3045 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
3046 }
3047
3048 if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
3049 if (chip_id == RTL8852A && cv == CHIP_CBV)
3050 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
3051 } else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
3052 if (chip_id == RTL8852A || chip_id == RTL8852B)
3053 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
3054 }
3055
3056 if (rxbd_mode == MAC_AX_RXBD_PKT) {
3057 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
3058 } else if (rxbd_mode == MAC_AX_RXBD_SEP) {
3059 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
3060
3061 if (chip_id == RTL8852A || chip_id == RTL8852B)
3062 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
3063 B_AX_PCIE_RX_APPLEN_MASK, 0);
3064 }
3065
3066 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3067 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
3068 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
3069 } else if (chip_id == RTL8852C) {
3070 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
3071 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
3072 }
3073
3074 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3075 if (tag_mode == MAC_AX_TAG_SGL) {
3076 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
3077 ~B_AX_LATENCY_CONTROL;
3078 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
3079 } else if (tag_mode == MAC_AX_TAG_MULTI) {
3080 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
3081 B_AX_LATENCY_CONTROL;
3082 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
3083 }
3084 }
3085
3086 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
3087 info->multi_tag_num);
3088
3089 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3090 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
3091 wd_dma_idle_intvl);
3092 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
3093 wd_dma_act_intvl);
3094 } else if (chip_id == RTL8852C) {
3095 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
3096 wd_dma_idle_intvl);
3097 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
3098 wd_dma_act_intvl);
3099 }
3100
3101 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
3102 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
3103 B_AX_HOST_ADDR_INFO_8B_SEL);
3104 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
3105 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
3106 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
3107 B_AX_HOST_ADDR_INFO_8B_SEL);
3108 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
3109 }
3110
3111 return 0;
3112 }
3113
rtw89_pci_ops_deinit(struct rtw89_dev * rtwdev)3114 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
3115 {
3116 const struct rtw89_pci_info *info = rtwdev->pci_info;
3117
3118 rtw89_pci_power_wake(rtwdev, false);
3119
3120 if (rtwdev->chip->chip_id == RTL8852A) {
3121 /* ltr sw trigger */
3122 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
3123 }
3124 info->ltr_set(rtwdev, false);
3125 rtw89_pci_ctrl_dma_all(rtwdev, false);
3126 rtw89_pci_clr_idx_all(rtwdev);
3127
3128 return 0;
3129 }
3130
rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev * rtwdev)3131 static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
3132 {
3133 const struct rtw89_pci_info *info = rtwdev->pci_info;
3134 int ret;
3135
3136 rtw89_pci_ber(rtwdev);
3137 rtw89_pci_rxdma_prefth(rtwdev);
3138 rtw89_pci_l1off_pwroff(rtwdev);
3139 rtw89_pci_deglitch_setting(rtwdev);
3140 ret = rtw89_pci_l2_rxen_lat(rtwdev);
3141 if (ret) {
3142 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
3143 return ret;
3144 }
3145
3146 rtw89_pci_aphy_pwrcut(rtwdev);
3147 rtw89_pci_hci_ldo(rtwdev);
3148 rtw89_pci_dphy_delay(rtwdev);
3149
3150 ret = rtw89_pci_autok_x(rtwdev);
3151 if (ret) {
3152 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
3153 return ret;
3154 }
3155
3156 ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
3157 if (ret) {
3158 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
3159 return ret;
3160 }
3161
3162 rtw89_pci_power_wake_ax(rtwdev, true);
3163 rtw89_pci_autoload_hang(rtwdev);
3164 rtw89_pci_l12_vmain(rtwdev);
3165 rtw89_pci_gen2_force_ib(rtwdev);
3166 rtw89_pci_l1_ent_lat(rtwdev);
3167 rtw89_pci_wd_exit_l1(rtwdev);
3168 rtw89_pci_set_sic(rtwdev);
3169 rtw89_pci_set_lbc(rtwdev);
3170 rtw89_pci_set_io_rcy(rtwdev);
3171 rtw89_pci_set_dbg(rtwdev);
3172 rtw89_pci_set_keep_reg(rtwdev);
3173
3174 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
3175
3176 /* stop DMA activities */
3177 rtw89_pci_ctrl_dma_all(rtwdev, false);
3178
3179 ret = rtw89_pci_poll_dma_all_idle(rtwdev);
3180 if (ret) {
3181 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
3182 return ret;
3183 }
3184
3185 rtw89_pci_clr_idx_all(rtwdev);
3186 rtw89_pci_mode_op(rtwdev);
3187
3188 /* fill TRX BD indexes */
3189 rtw89_pci_ops_reset(rtwdev);
3190
3191 ret = rtw89_pci_rst_bdram_ax(rtwdev);
3192 if (ret) {
3193 rtw89_warn(rtwdev, "reset bdram busy\n");
3194 return ret;
3195 }
3196
3197 /* disable all channels except to FW CMD channel to download firmware */
3198 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, false);
3199 rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, true);
3200
3201 /* start DMA activities */
3202 rtw89_pci_ctrl_dma_all(rtwdev, true);
3203
3204 return 0;
3205 }
3206
rtw89_pci_ops_mac_pre_deinit_ax(struct rtw89_dev * rtwdev)3207 static int rtw89_pci_ops_mac_pre_deinit_ax(struct rtw89_dev *rtwdev)
3208 {
3209 rtw89_pci_power_wake_ax(rtwdev, false);
3210
3211 return 0;
3212 }
3213
rtw89_pci_ltr_set(struct rtw89_dev * rtwdev,bool en)3214 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
3215 {
3216 u32 val;
3217
3218 if (!en)
3219 return 0;
3220
3221 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
3222 if (rtw89_pci_ltr_is_err_reg_val(val))
3223 return -EINVAL;
3224 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
3225 if (rtw89_pci_ltr_is_err_reg_val(val))
3226 return -EINVAL;
3227 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
3228 if (rtw89_pci_ltr_is_err_reg_val(val))
3229 return -EINVAL;
3230 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
3231 if (rtw89_pci_ltr_is_err_reg_val(val))
3232 return -EINVAL;
3233
3234 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
3235 B_AX_LTR_WD_NOEMP_CHK);
3236 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
3237 PCI_LTR_SPC_500US);
3238 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
3239 PCI_LTR_IDLE_TIMER_3_2MS);
3240 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
3241 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
3242 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
3243 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
3244
3245 return 0;
3246 }
3247 EXPORT_SYMBOL(rtw89_pci_ltr_set);
3248
rtw89_pci_ltr_set_v1(struct rtw89_dev * rtwdev,bool en)3249 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
3250 {
3251 u32 dec_ctrl;
3252 u32 val32;
3253
3254 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
3255 if (rtw89_pci_ltr_is_err_reg_val(val32))
3256 return -EINVAL;
3257 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
3258 if (rtw89_pci_ltr_is_err_reg_val(val32))
3259 return -EINVAL;
3260 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
3261 if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
3262 return -EINVAL;
3263 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
3264 if (rtw89_pci_ltr_is_err_reg_val(val32))
3265 return -EINVAL;
3266 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
3267 if (rtw89_pci_ltr_is_err_reg_val(val32))
3268 return -EINVAL;
3269
3270 if (!en) {
3271 dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
3272 dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
3273 B_AX_LTR_REQ_DRV;
3274 } else {
3275 dec_ctrl |= B_AX_LTR_HW_DEC_EN;
3276 }
3277
3278 dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
3279 dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
3280
3281 if (en)
3282 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
3283 B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
3284 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
3285 PCI_LTR_IDLE_TIMER_3_2MS);
3286 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
3287 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
3288 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
3289 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
3290 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
3291
3292 return 0;
3293 }
3294 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
3295
rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev * rtwdev)3296 static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
3297 {
3298 const struct rtw89_pci_info *info = rtwdev->pci_info;
3299 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3300 int ret;
3301
3302 ret = info->ltr_set(rtwdev, true);
3303 if (ret) {
3304 rtw89_err(rtwdev, "pci ltr set fail\n");
3305 return ret;
3306 }
3307 if (chip_id == RTL8852A) {
3308 /* ltr sw trigger */
3309 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
3310 }
3311 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3312 /* ADDR info 8-byte mode */
3313 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
3314 B_AX_HOST_ADDR_INFO_8B_SEL);
3315 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
3316 }
3317
3318 /* enable DMA for all queues */
3319 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, true);
3320
3321 /* Release PCI IO */
3322 rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
3323 B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
3324
3325 return 0;
3326 }
3327
rtw89_pci_claim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3328 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
3329 struct pci_dev *pdev)
3330 {
3331 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3332 int ret;
3333
3334 ret = pci_enable_device(pdev);
3335 if (ret) {
3336 rtw89_err(rtwdev, "failed to enable pci device\n");
3337 return ret;
3338 }
3339
3340 pci_set_master(pdev);
3341 pci_set_drvdata(pdev, rtwdev->hw);
3342
3343 rtwpci->pdev = pdev;
3344
3345 return 0;
3346 }
3347
rtw89_pci_declaim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3348 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
3349 struct pci_dev *pdev)
3350 {
3351 pci_disable_device(pdev);
3352 }
3353
rtw89_pci_chip_is_manual_dac(struct rtw89_dev * rtwdev)3354 static bool rtw89_pci_chip_is_manual_dac(struct rtw89_dev *rtwdev)
3355 {
3356 const struct rtw89_chip_info *chip = rtwdev->chip;
3357
3358 switch (chip->chip_id) {
3359 case RTL8852A:
3360 case RTL8852B:
3361 case RTL8851B:
3362 case RTL8852BT:
3363 return true;
3364 default:
3365 return false;
3366 }
3367 }
3368
rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev * rtwdev)3369 static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev)
3370 {
3371 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3372 struct pci_dev *bridge = pci_upstream_bridge(rtwpci->pdev);
3373
3374 if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3375 return true;
3376
3377 if (!bridge)
3378 return false;
3379
3380 switch (bridge->vendor) {
3381 case PCI_VENDOR_ID_INTEL:
3382 return true;
3383 case PCI_VENDOR_ID_ASMEDIA:
3384 if (bridge->device == 0x2806)
3385 return true;
3386 break;
3387 }
3388
3389 return false;
3390 }
3391
rtw89_pci_cfg_dac(struct rtw89_dev * rtwdev,bool force)3392 static int rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev, bool force)
3393 {
3394 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3395 struct pci_dev *pdev = rtwpci->pdev;
3396 int ret;
3397 u8 val;
3398
3399 if (!rtwpci->enable_dac && !force)
3400 return 0;
3401
3402 if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3403 return 0;
3404
3405 /* Configure DAC only via PCI config API, not DBI interfaces */
3406 ret = pci_read_config_byte(pdev, RTW89_PCIE_L1_CTRL, &val);
3407 if (ret)
3408 return ret;
3409
3410 val |= RTW89_PCIE_BIT_EN_64BITS;
3411 return pci_write_config_byte(pdev, RTW89_PCIE_L1_CTRL, val);
3412 }
3413
rtw89_pci_setup_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3414 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
3415 struct pci_dev *pdev)
3416 {
3417 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3418 unsigned long resource_len;
3419 u8 bar_id = 2;
3420 int ret;
3421
3422 ret = pci_request_regions(pdev, KBUILD_MODNAME);
3423 if (ret) {
3424 rtw89_err(rtwdev, "failed to request pci regions\n");
3425 goto err;
3426 }
3427
3428 if (!rtw89_pci_is_dac_compatible_bridge(rtwdev))
3429 goto try_dac_done;
3430
3431 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36));
3432 if (!ret) {
3433 ret = rtw89_pci_cfg_dac(rtwdev, true);
3434 if (!ret) {
3435 rtwpci->enable_dac = true;
3436 goto try_dac_done;
3437 }
3438
3439 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3440 if (ret) {
3441 rtw89_err(rtwdev,
3442 "failed to set dma and consistent mask to 32/36-bit\n");
3443 goto err_release_regions;
3444 }
3445 }
3446 try_dac_done:
3447
3448 #if defined(__FreeBSD__)
3449 linuxkpi_pcim_want_to_use_bus_functions(pdev);
3450 #endif
3451 resource_len = pci_resource_len(pdev, bar_id);
3452 rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len);
3453 if (!rtwpci->mmap) {
3454 rtw89_err(rtwdev, "failed to map pci io\n");
3455 ret = -EIO;
3456 goto err_release_regions;
3457 }
3458
3459 return 0;
3460
3461 err_release_regions:
3462 pci_release_regions(pdev);
3463 err:
3464 return ret;
3465 }
3466
rtw89_pci_clear_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3467 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
3468 struct pci_dev *pdev)
3469 {
3470 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3471
3472 if (rtwpci->mmap) {
3473 pci_iounmap(pdev, rtwpci->mmap);
3474 pci_release_regions(pdev);
3475 }
3476 }
3477
rtw89_pci_free_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)3478 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
3479 struct pci_dev *pdev,
3480 struct rtw89_pci_tx_ring *tx_ring)
3481 {
3482 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
3483 u8 *head = wd_ring->head;
3484 dma_addr_t dma = wd_ring->dma;
3485 u32 page_size = wd_ring->page_size;
3486 u32 page_num = wd_ring->page_num;
3487 u32 ring_sz = page_size * page_num;
3488
3489 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3490 wd_ring->head = NULL;
3491 }
3492
rtw89_pci_free_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)3493 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
3494 struct pci_dev *pdev,
3495 struct rtw89_pci_tx_ring *tx_ring)
3496 {
3497 tx_ring->bd_ring.head = NULL;
3498 }
3499
rtw89_pci_free_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3500 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
3501 struct pci_dev *pdev)
3502 {
3503 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3504 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->tx.bd_pool;
3505 const struct rtw89_pci_info *info = rtwdev->pci_info;
3506 struct rtw89_pci_tx_ring *tx_ring;
3507 int i;
3508
3509 for (i = 0; i < RTW89_TXCH_NUM; i++) {
3510 if (info->tx_dma_ch_mask & BIT(i))
3511 continue;
3512 tx_ring = &rtwpci->tx.rings[i];
3513 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3514 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3515 }
3516
3517 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma);
3518 }
3519
rtw89_pci_free_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring)3520 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
3521 struct pci_dev *pdev,
3522 struct rtw89_pci_rx_ring *rx_ring)
3523 {
3524 struct rtw89_pci_rx_info *rx_info;
3525 struct sk_buff *skb;
3526 dma_addr_t dma;
3527 u32 buf_sz;
3528 int i;
3529
3530 buf_sz = rx_ring->buf_sz;
3531 for (i = 0; i < rx_ring->bd_ring.len; i++) {
3532 skb = rx_ring->buf[i];
3533 if (!skb)
3534 continue;
3535
3536 rx_info = RTW89_PCI_RX_SKB_CB(skb);
3537 dma = rx_info->dma;
3538 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3539 dev_kfree_skb(skb);
3540 rx_ring->buf[i] = NULL;
3541 }
3542
3543 rx_ring->bd_ring.head = NULL;
3544 }
3545
rtw89_pci_free_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3546 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
3547 struct pci_dev *pdev)
3548 {
3549 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3550 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->rx.bd_pool;
3551 struct rtw89_pci_rx_ring *rx_ring;
3552 int i;
3553
3554 for (i = 0; i < RTW89_RXCH_NUM; i++) {
3555 rx_ring = &rtwpci->rx.rings[i];
3556 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3557 }
3558
3559 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma);
3560 }
3561
rtw89_pci_free_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3562 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
3563 struct pci_dev *pdev)
3564 {
3565 rtw89_pci_free_rx_rings(rtwdev, pdev);
3566 rtw89_pci_free_tx_rings(rtwdev, pdev);
3567 }
3568
rtw89_pci_init_rx_bd(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb,int buf_sz,u32 idx)3569 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
3570 struct rtw89_pci_rx_ring *rx_ring,
3571 struct sk_buff *skb, int buf_sz, u32 idx)
3572 {
3573 struct rtw89_pci_rx_info *rx_info;
3574 struct rtw89_pci_rx_bd_32 *rx_bd;
3575 dma_addr_t dma;
3576
3577 if (!skb)
3578 return -EINVAL;
3579
3580 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
3581 if (dma_mapping_error(&pdev->dev, dma))
3582 return -EBUSY;
3583
3584 rx_info = RTW89_PCI_RX_SKB_CB(skb);
3585 rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
3586
3587 memset(rx_bd, 0, sizeof(*rx_bd));
3588 rx_bd->buf_size = cpu_to_le16(buf_sz);
3589 rx_bd->dma = cpu_to_le32(dma);
3590 rx_bd->opt = le16_encode_bits(upper_32_bits(dma), RTW89_PCI_RXBD_OPT_DMA_HI);
3591 rx_info->dma = dma;
3592
3593 return 0;
3594 }
3595
rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,enum rtw89_tx_channel txch)3596 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
3597 struct pci_dev *pdev,
3598 struct rtw89_pci_tx_ring *tx_ring,
3599 enum rtw89_tx_channel txch)
3600 {
3601 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
3602 struct rtw89_pci_tx_wd *txwd;
3603 dma_addr_t dma;
3604 dma_addr_t cur_paddr;
3605 u8 *head;
3606 u8 *cur_vaddr;
3607 u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
3608 u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
3609 u32 ring_sz = page_size * page_num;
3610 u32 page_offset;
3611 int i;
3612
3613 /* FWCMD queue doesn't use txwd as pages */
3614 if (txch == RTW89_TXCH_CH12)
3615 return 0;
3616
3617 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3618 if (!head)
3619 return -ENOMEM;
3620
3621 INIT_LIST_HEAD(&wd_ring->free_pages);
3622 wd_ring->head = head;
3623 wd_ring->dma = dma;
3624 wd_ring->page_size = page_size;
3625 wd_ring->page_num = page_num;
3626
3627 page_offset = 0;
3628 for (i = 0; i < page_num; i++) {
3629 txwd = &wd_ring->pages[i];
3630 cur_paddr = dma + page_offset;
3631 cur_vaddr = head + page_offset;
3632
3633 skb_queue_head_init(&txwd->queue);
3634 INIT_LIST_HEAD(&txwd->list);
3635 txwd->paddr = cur_paddr;
3636 txwd->vaddr = cur_vaddr;
3637 txwd->len = page_size;
3638 txwd->seq = i;
3639 rtw89_pci_enqueue_txwd(tx_ring, txwd);
3640
3641 page_offset += page_size;
3642 }
3643
3644 return 0;
3645 }
3646
rtw89_pci_alloc_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,u32 desc_size,u32 len,enum rtw89_tx_channel txch,void * head,dma_addr_t dma)3647 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
3648 struct pci_dev *pdev,
3649 struct rtw89_pci_tx_ring *tx_ring,
3650 u32 desc_size, u32 len,
3651 enum rtw89_tx_channel txch,
3652 void *head, dma_addr_t dma)
3653 {
3654 const struct rtw89_pci_ch_dma_addr *txch_addr;
3655 int ret;
3656
3657 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
3658 if (ret) {
3659 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
3660 goto err;
3661 }
3662
3663 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
3664 if (ret) {
3665 rtw89_err(rtwdev, "failed to get address of txch %d", txch);
3666 goto err_free_wd_ring;
3667 }
3668
3669 INIT_LIST_HEAD(&tx_ring->busy_pages);
3670 tx_ring->bd_ring.head = head;
3671 tx_ring->bd_ring.dma = dma;
3672 tx_ring->bd_ring.len = len;
3673 tx_ring->bd_ring.desc_size = desc_size;
3674 tx_ring->bd_ring.addr = *txch_addr;
3675 tx_ring->bd_ring.wp = 0;
3676 tx_ring->bd_ring.rp = 0;
3677 tx_ring->txch = txch;
3678
3679 return 0;
3680
3681 err_free_wd_ring:
3682 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3683 err:
3684 return ret;
3685 }
3686
rtw89_pci_alloc_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3687 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
3688 struct pci_dev *pdev)
3689 {
3690 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3691 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->tx.bd_pool;
3692 const struct rtw89_pci_info *info = rtwdev->pci_info;
3693 struct rtw89_pci_tx_ring *tx_ring;
3694 u32 i, tx_allocated;
3695 dma_addr_t dma;
3696 u32 desc_size;
3697 u32 ring_sz;
3698 u32 pool_sz;
3699 u32 ch_num;
3700 #if defined(__linux__)
3701 void *head;
3702 #elif defined(__FreeBSD__)
3703 u8 *head;
3704 #endif
3705 u32 len;
3706 int ret;
3707
3708 BUILD_BUG_ON(RTW89_PCI_TXBD_NUM_MAX % 16);
3709
3710 desc_size = sizeof(struct rtw89_pci_tx_bd_32);
3711 len = RTW89_PCI_TXBD_NUM_MAX;
3712 ch_num = RTW89_TXCH_NUM - hweight32(info->tx_dma_ch_mask);
3713 ring_sz = desc_size * len;
3714 pool_sz = ring_sz * ch_num;
3715
3716 head = dma_alloc_coherent(&pdev->dev, pool_sz, &dma, GFP_KERNEL);
3717 if (!head)
3718 return -ENOMEM;
3719
3720 bd_pool->head = head;
3721 bd_pool->dma = dma;
3722 bd_pool->size = pool_sz;
3723
3724 for (i = 0; i < RTW89_TXCH_NUM; i++) {
3725 if (info->tx_dma_ch_mask & BIT(i))
3726 continue;
3727 tx_ring = &rtwpci->tx.rings[i];
3728 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3729 desc_size, len, i, head, dma);
3730 if (ret) {
3731 #if defined(__linux__)
3732 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3733 #elif defined(__FreeBSD__)
3734 rtw89_err(rtwdev, "failed to alloc tx ring %d: ret=%d\n", i, ret);
3735 #endif
3736 goto err_free;
3737 }
3738
3739 head += ring_sz;
3740 dma += ring_sz;
3741 }
3742
3743 return 0;
3744
3745 err_free:
3746 tx_allocated = i;
3747 for (i = 0; i < tx_allocated; i++) {
3748 tx_ring = &rtwpci->tx.rings[i];
3749 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3750 }
3751
3752 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma);
3753
3754 return ret;
3755 }
3756
rtw89_pci_alloc_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,u32 desc_size,u32 len,u32 rxch,void * head,dma_addr_t dma)3757 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3758 struct pci_dev *pdev,
3759 struct rtw89_pci_rx_ring *rx_ring,
3760 u32 desc_size, u32 len, u32 rxch,
3761 void *head, dma_addr_t dma)
3762 {
3763 const struct rtw89_pci_info *info = rtwdev->pci_info;
3764 const struct rtw89_pci_ch_dma_addr *rxch_addr;
3765 struct sk_buff *skb;
3766 int buf_sz = RTW89_PCI_RX_BUF_SIZE;
3767 int i, allocated;
3768 int ret;
3769
3770 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3771 if (ret) {
3772 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3773 return ret;
3774 }
3775
3776 rx_ring->bd_ring.head = head;
3777 rx_ring->bd_ring.dma = dma;
3778 rx_ring->bd_ring.len = len;
3779 rx_ring->bd_ring.desc_size = desc_size;
3780 rx_ring->bd_ring.addr = *rxch_addr;
3781 if (info->rx_ring_eq_is_full)
3782 rx_ring->bd_ring.wp = len - 1;
3783 else
3784 rx_ring->bd_ring.wp = 0;
3785 rx_ring->bd_ring.rp = 0;
3786 rx_ring->buf_sz = buf_sz;
3787 rx_ring->diliver_skb = NULL;
3788 rx_ring->diliver_desc.ready = false;
3789 rx_ring->target_rx_tag = 0;
3790
3791 for (i = 0; i < len; i++) {
3792 skb = dev_alloc_skb(buf_sz);
3793 if (!skb) {
3794 ret = -ENOMEM;
3795 goto err_free;
3796 }
3797
3798 memset(skb->data, 0, buf_sz);
3799 rx_ring->buf[i] = skb;
3800 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3801 buf_sz, i);
3802 if (ret) {
3803 #if defined(__linux__)
3804 rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3805 #elif defined(__FreeBSD__)
3806 rtw89_err(rtwdev, "failed to init rx buf %d ret=%d\n", i, ret);
3807 #endif
3808 dev_kfree_skb_any(skb);
3809 rx_ring->buf[i] = NULL;
3810 goto err_free;
3811 }
3812 }
3813
3814 return 0;
3815
3816 err_free:
3817 allocated = i;
3818 for (i = 0; i < allocated; i++) {
3819 skb = rx_ring->buf[i];
3820 if (!skb)
3821 continue;
3822 dma = *((dma_addr_t *)skb->cb);
3823 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3824 dev_kfree_skb(skb);
3825 rx_ring->buf[i] = NULL;
3826 }
3827
3828 rx_ring->bd_ring.head = NULL;
3829
3830 return ret;
3831 }
3832
rtw89_pci_alloc_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3833 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3834 struct pci_dev *pdev)
3835 {
3836 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3837 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->rx.bd_pool;
3838 struct rtw89_pci_rx_ring *rx_ring;
3839 int i, rx_allocated;
3840 dma_addr_t dma;
3841 u32 desc_size;
3842 u32 ring_sz;
3843 u32 pool_sz;
3844 #if defined(__linux__)
3845 void *head;
3846 #elif defined(__FreeBSD__)
3847 u8 *head;
3848 #endif
3849 u32 len;
3850 int ret;
3851
3852 desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3853 len = RTW89_PCI_RXBD_NUM_MAX;
3854 ring_sz = desc_size * len;
3855 pool_sz = ring_sz * RTW89_RXCH_NUM;
3856
3857 head = dma_alloc_coherent(&pdev->dev, pool_sz, &dma, GFP_KERNEL);
3858 if (!head)
3859 return -ENOMEM;
3860
3861 bd_pool->head = head;
3862 bd_pool->dma = dma;
3863 bd_pool->size = pool_sz;
3864
3865 for (i = 0; i < RTW89_RXCH_NUM; i++) {
3866 rx_ring = &rtwpci->rx.rings[i];
3867
3868 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3869 desc_size, len, i,
3870 head, dma);
3871 if (ret) {
3872 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3873 goto err_free;
3874 }
3875
3876 head += ring_sz;
3877 dma += ring_sz;
3878 }
3879
3880 return 0;
3881
3882 err_free:
3883 rx_allocated = i;
3884 for (i = 0; i < rx_allocated; i++) {
3885 rx_ring = &rtwpci->rx.rings[i];
3886 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3887 }
3888
3889 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma);
3890
3891 return ret;
3892 }
3893
rtw89_pci_alloc_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3894 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3895 struct pci_dev *pdev)
3896 {
3897 int ret;
3898
3899 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3900 if (ret) {
3901 rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3902 goto err;
3903 }
3904
3905 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3906 if (ret) {
3907 rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3908 goto err_free_tx_rings;
3909 }
3910
3911 return 0;
3912
3913 err_free_tx_rings:
3914 rtw89_pci_free_tx_rings(rtwdev, pdev);
3915 err:
3916 return ret;
3917 }
3918
rtw89_pci_h2c_init(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)3919 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3920 struct rtw89_pci *rtwpci)
3921 {
3922 skb_queue_head_init(&rtwpci->h2c_queue);
3923 skb_queue_head_init(&rtwpci->h2c_release_queue);
3924 }
3925
rtw89_pci_setup_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3926 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3927 struct pci_dev *pdev)
3928 {
3929 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3930 int ret;
3931
3932 ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3933 if (ret) {
3934 rtw89_err(rtwdev, "failed to setup pci mapping\n");
3935 goto err;
3936 }
3937
3938 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3939 if (ret) {
3940 rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3941 goto err_pci_unmap;
3942 }
3943
3944 rtw89_pci_h2c_init(rtwdev, rtwpci);
3945
3946 spin_lock_init(&rtwpci->irq_lock);
3947 spin_lock_init(&rtwpci->trx_lock);
3948
3949 return 0;
3950
3951 err_pci_unmap:
3952 rtw89_pci_clear_mapping(rtwdev, pdev);
3953 err:
3954 return ret;
3955 }
3956
rtw89_pci_clear_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3957 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3958 struct pci_dev *pdev)
3959 {
3960 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3961
3962 rtw89_pci_free_trx_rings(rtwdev, pdev);
3963 rtw89_pci_clear_mapping(rtwdev, pdev);
3964 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3965 skb_queue_len(&rtwpci->h2c_queue), true);
3966 }
3967
rtw89_pci_config_intr_mask(struct rtw89_dev * rtwdev)3968 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3969 {
3970 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3971 const struct rtw89_chip_info *chip = rtwdev->chip;
3972 u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN;
3973
3974 if (chip->chip_id == RTL8851B)
3975 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND;
3976
3977 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3978
3979 if (rtwpci->under_recovery) {
3980 rtwpci->intrs[0] = hs0isr_ind_int_en;
3981 rtwpci->intrs[1] = 0;
3982 } else {
3983 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3984 B_AX_RXDMA_INT_EN |
3985 B_AX_RXP1DMA_INT_EN |
3986 B_AX_RPQDMA_INT_EN |
3987 B_AX_RXDMA_STUCK_INT_EN |
3988 B_AX_RDU_INT_EN |
3989 B_AX_RPQBD_FULL_INT_EN |
3990 hs0isr_ind_int_en;
3991
3992 rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3993 }
3994 }
3995 EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3996
rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev * rtwdev)3997 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3998 {
3999 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4000
4001 rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
4002 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
4003 rtwpci->intrs[0] = 0;
4004 rtwpci->intrs[1] = 0;
4005 }
4006
rtw89_pci_default_intr_mask_v1(struct rtw89_dev * rtwdev)4007 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
4008 {
4009 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4010
4011 rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
4012 B_AX_HS1ISR_IND_INT_EN |
4013 B_AX_HS0ISR_IND_INT_EN;
4014 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
4015 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
4016 B_AX_RXDMA_INT_EN |
4017 B_AX_RXP1DMA_INT_EN |
4018 B_AX_RPQDMA_INT_EN |
4019 B_AX_RXDMA_STUCK_INT_EN |
4020 B_AX_RDU_INT_EN |
4021 B_AX_RPQBD_FULL_INT_EN;
4022 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
4023 }
4024
rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev * rtwdev)4025 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
4026 {
4027 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4028
4029 rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
4030 B_AX_HS0ISR_IND_INT_EN;
4031 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
4032 rtwpci->intrs[0] = 0;
4033 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
4034 }
4035
rtw89_pci_config_intr_mask_v1(struct rtw89_dev * rtwdev)4036 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
4037 {
4038 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4039
4040 if (rtwpci->under_recovery)
4041 rtw89_pci_recovery_intr_mask_v1(rtwdev);
4042 else if (rtwpci->low_power)
4043 rtw89_pci_low_power_intr_mask_v1(rtwdev);
4044 else
4045 rtw89_pci_default_intr_mask_v1(rtwdev);
4046 }
4047 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
4048
rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev * rtwdev)4049 static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
4050 {
4051 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4052
4053 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
4054 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
4055 rtwpci->intrs[0] = 0;
4056 rtwpci->intrs[1] = 0;
4057 }
4058
rtw89_pci_default_intr_mask_v2(struct rtw89_dev * rtwdev)4059 static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
4060 {
4061 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4062
4063 rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 |
4064 B_BE_HS0_IND_INT_EN0;
4065 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
4066 rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 |
4067 B_BE_RDU_CH0_INT_IMR_V1;
4068 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
4069 B_BE_PCIE_RX_RPQ0_IMR0_V1;
4070 }
4071
rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev * rtwdev)4072 static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
4073 {
4074 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4075
4076 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 |
4077 B_BE_HS1_IND_INT_EN0;
4078 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
4079 rtwpci->intrs[0] = 0;
4080 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
4081 B_BE_PCIE_RX_RPQ0_IMR0_V1;
4082 }
4083
rtw89_pci_config_intr_mask_v2(struct rtw89_dev * rtwdev)4084 void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
4085 {
4086 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4087
4088 if (rtwpci->under_recovery)
4089 rtw89_pci_recovery_intr_mask_v2(rtwdev);
4090 else if (rtwpci->low_power)
4091 rtw89_pci_low_power_intr_mask_v2(rtwdev);
4092 else
4093 rtw89_pci_default_intr_mask_v2(rtwdev);
4094 }
4095 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2);
4096
rtw89_pci_recovery_intr_mask_v3(struct rtw89_dev * rtwdev)4097 static void rtw89_pci_recovery_intr_mask_v3(struct rtw89_dev *rtwdev)
4098 {
4099 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4100
4101 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
4102 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN |
4103 B_BE_SPSANA_OCP_INT_EN | B_BE_SPS_OCP_INT_EN;
4104 rtwpci->intrs[0] = 0;
4105 rtwpci->intrs[1] = 0;
4106 }
4107
rtw89_pci_default_intr_mask_v3(struct rtw89_dev * rtwdev)4108 static void rtw89_pci_default_intr_mask_v3(struct rtw89_dev *rtwdev)
4109 {
4110 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4111
4112 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
4113 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN |
4114 B_BE_SPSANA_OCP_INT_EN | B_BE_SPS_OCP_INT_EN;
4115 rtwpci->intrs[0] = 0;
4116 rtwpci->intrs[1] = B_BE_PCIE_RDU_CH1_IMR |
4117 B_BE_PCIE_RDU_CH0_IMR |
4118 B_BE_PCIE_RX_RX0P2_IMR0_V1 |
4119 B_BE_PCIE_RX_RPQ0_IMR0_V1;
4120 }
4121
rtw89_pci_config_intr_mask_v3(struct rtw89_dev * rtwdev)4122 void rtw89_pci_config_intr_mask_v3(struct rtw89_dev *rtwdev)
4123 {
4124 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4125
4126 if (rtwpci->under_recovery)
4127 rtw89_pci_recovery_intr_mask_v3(rtwdev);
4128 else
4129 rtw89_pci_default_intr_mask_v3(rtwdev);
4130 }
4131 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v3);
4132
rtw89_pci_request_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)4133 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
4134 struct pci_dev *pdev)
4135 {
4136 unsigned long flags = 0;
4137 int ret;
4138
4139 flags |= PCI_IRQ_INTX | PCI_IRQ_MSI;
4140 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
4141 if (ret < 0) {
4142 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
4143 goto err;
4144 }
4145
4146 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
4147 rtw89_pci_interrupt_handler,
4148 rtw89_pci_interrupt_threadfn,
4149 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
4150 if (ret) {
4151 rtw89_err(rtwdev, "failed to request threaded irq\n");
4152 goto err_free_vector;
4153 }
4154
4155 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
4156
4157 return 0;
4158
4159 err_free_vector:
4160 pci_free_irq_vectors(pdev);
4161 err:
4162 return ret;
4163 }
4164
rtw89_pci_free_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)4165 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
4166 struct pci_dev *pdev)
4167 {
4168 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
4169 pci_free_irq_vectors(pdev);
4170 }
4171
gray_code_to_bin(u16 gray_code)4172 static u16 gray_code_to_bin(u16 gray_code)
4173 {
4174 u16 binary = gray_code;
4175
4176 while (gray_code) {
4177 gray_code >>= 1;
4178 binary ^= gray_code;
4179 }
4180
4181 return binary;
4182 }
4183
rtw89_pci_filter_out(struct rtw89_dev * rtwdev)4184 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
4185 {
4186 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4187 struct pci_dev *pdev = rtwpci->pdev;
4188 u16 val16, filter_out_val;
4189 u32 val, phy_offset;
4190 int ret;
4191
4192 if (rtwdev->chip->chip_id != RTL8852C)
4193 return 0;
4194
4195 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
4196 if (val == B_AX_ASPM_CTRL_L1)
4197 return 0;
4198
4199 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
4200 if (ret)
4201 return ret;
4202
4203 val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
4204 if (val == RTW89_PCIE_GEN1_SPEED) {
4205 phy_offset = R_RAC_DIRECT_OFFSET_G1;
4206 } else if (val == RTW89_PCIE_GEN2_SPEED) {
4207 phy_offset = R_RAC_DIRECT_OFFSET_G2;
4208 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
4209 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
4210 val16 | B_PCIE_BIT_PINOUT_DIS);
4211 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
4212 val16 & ~B_PCIE_BIT_RD_SEL);
4213
4214 val16 = rtw89_read16_mask(rtwdev,
4215 phy_offset + RAC_ANA1F * RAC_MULT,
4216 FILTER_OUT_EQ_MASK);
4217 val16 = gray_code_to_bin(val16);
4218 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
4219 RAC_MULT);
4220 filter_out_val &= ~REG_FILTER_OUT_MASK;
4221 filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
4222
4223 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
4224 filter_out_val);
4225 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
4226 B_BAC_EQ_SEL);
4227 rtw89_write16_set(rtwdev,
4228 R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
4229 B_PCIE_BIT_PSAVE);
4230 } else {
4231 return -EOPNOTSUPP;
4232 }
4233 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
4234 B_PCIE_BIT_PSAVE);
4235
4236 return 0;
4237 }
4238
rtw89_pci_clkreq_set(struct rtw89_dev * rtwdev,bool enable)4239 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
4240 {
4241 const struct rtw89_pci_info *info = rtwdev->pci_info;
4242 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4243
4244 if (rtw89_pci_disable_clkreq)
4245 return;
4246
4247 gen_def->clkreq_set(rtwdev, enable);
4248 }
4249
rtw89_pci_clkreq_set_ax(struct rtw89_dev * rtwdev,bool enable)4250 static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable)
4251 {
4252 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4253 int ret;
4254
4255 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
4256 PCIE_CLKDLY_HW_30US);
4257 if (ret)
4258 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
4259
4260 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4261 if (enable)
4262 ret = rtw89_pci_config_byte_set(rtwdev,
4263 RTW89_PCIE_L1_CTRL,
4264 RTW89_PCIE_BIT_CLK);
4265 else
4266 ret = rtw89_pci_config_byte_clr(rtwdev,
4267 RTW89_PCIE_L1_CTRL,
4268 RTW89_PCIE_BIT_CLK);
4269 if (ret)
4270 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
4271 enable ? "set" : "unset", ret);
4272 } else if (chip_id == RTL8852C) {
4273 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
4274 B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
4275 if (enable)
4276 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
4277 B_AX_CLK_REQ_N);
4278 else
4279 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
4280 B_AX_CLK_REQ_N);
4281 }
4282 }
4283
rtw89_pci_aspm_set(struct rtw89_dev * rtwdev,bool enable)4284 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
4285 {
4286 const struct rtw89_pci_info *info = rtwdev->pci_info;
4287 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4288
4289 if (rtw89_pci_disable_aspm_l1)
4290 return;
4291
4292 gen_def->aspm_set(rtwdev, enable);
4293 }
4294
rtw89_pci_aspm_set_ax(struct rtw89_dev * rtwdev,bool enable)4295 static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable)
4296 {
4297 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4298 u8 value = 0;
4299 int ret;
4300
4301 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
4302 if (ret)
4303 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
4304
4305 u8p_replace_bits(&value, PCIE_L1DLY_16US, RTW89_L1DLY_MASK);
4306 u8p_replace_bits(&value, PCIE_L0SDLY_4US, RTW89_L0DLY_MASK);
4307
4308 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
4309 if (ret)
4310 rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
4311
4312 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4313 if (enable)
4314 ret = rtw89_pci_config_byte_set(rtwdev,
4315 RTW89_PCIE_L1_CTRL,
4316 RTW89_PCIE_BIT_L1);
4317 else
4318 ret = rtw89_pci_config_byte_clr(rtwdev,
4319 RTW89_PCIE_L1_CTRL,
4320 RTW89_PCIE_BIT_L1);
4321 } else if (chip_id == RTL8852C) {
4322 if (enable)
4323 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4324 B_AX_ASPM_CTRL_L1);
4325 else
4326 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4327 B_AX_ASPM_CTRL_L1);
4328 }
4329 if (ret)
4330 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
4331 enable ? "set" : "unset", ret);
4332 }
4333
rtw89_pci_recalc_int_mit(struct rtw89_dev * rtwdev)4334 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
4335 {
4336 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
4337 const struct rtw89_pci_info *info = rtwdev->pci_info;
4338 struct rtw89_traffic_stats *stats = &rtwdev->stats;
4339 enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
4340 enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
4341 u32 val = 0;
4342
4343 if (rtwdev->scanning ||
4344 (tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH))
4345 goto out;
4346
4347 if (chip_gen == RTW89_CHIP_BE)
4348 val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN;
4349 else
4350 val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
4351 FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
4352 FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
4353 FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
4354
4355 out:
4356 rtw89_write32(rtwdev, info->mit_addr, val);
4357 }
4358
rtw89_pci_link_cfg(struct rtw89_dev * rtwdev)4359 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
4360 {
4361 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4362 struct pci_dev *pdev = rtwpci->pdev;
4363 u16 link_ctrl;
4364 int ret;
4365
4366 /* Though there is standard PCIE configuration space to set the
4367 * link control register, but by Realtek's design, driver should
4368 * check if host supports CLKREQ/ASPM to enable the HW module.
4369 *
4370 * These functions are implemented by two HW modules associated,
4371 * one is responsible to access PCIE configuration space to
4372 * follow the host settings, and another is in charge of doing
4373 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
4374 * the host does not support it, and due to some reasons or wrong
4375 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
4376 * loss if HW misbehaves on the link.
4377 *
4378 * Hence it's designed that driver should first check the PCIE
4379 * configuration space is sync'ed and enabled, then driver can turn
4380 * on the other module that is actually working on the mechanism.
4381 */
4382 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
4383 if (ret) {
4384 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
4385 return;
4386 }
4387
4388 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
4389 rtw89_pci_clkreq_set(rtwdev, true);
4390
4391 if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
4392 rtw89_pci_aspm_set(rtwdev, true);
4393 }
4394
rtw89_pci_l1ss_set(struct rtw89_dev * rtwdev,bool enable)4395 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
4396 {
4397 const struct rtw89_pci_info *info = rtwdev->pci_info;
4398 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4399
4400 if (rtw89_pci_disable_l1ss)
4401 return;
4402
4403 gen_def->l1ss_set(rtwdev, enable);
4404 }
4405
rtw89_pci_l1ss_set_ax(struct rtw89_dev * rtwdev,bool enable)4406 static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable)
4407 {
4408 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4409 int ret;
4410
4411 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4412 if (enable)
4413 ret = rtw89_pci_config_byte_set(rtwdev,
4414 RTW89_PCIE_TIMER_CTRL,
4415 RTW89_PCIE_BIT_L1SUB);
4416 else
4417 ret = rtw89_pci_config_byte_clr(rtwdev,
4418 RTW89_PCIE_TIMER_CTRL,
4419 RTW89_PCIE_BIT_L1SUB);
4420 if (ret)
4421 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
4422 enable ? "set" : "unset", ret);
4423 } else if (chip_id == RTL8852C) {
4424 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
4425 RTW89_PCIE_BIT_ASPM_L11 |
4426 RTW89_PCIE_BIT_PCI_L11);
4427 if (ret)
4428 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
4429 if (enable)
4430 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4431 B_AX_L1SUB_DISABLE);
4432 else
4433 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4434 B_AX_L1SUB_DISABLE);
4435 }
4436 }
4437
rtw89_pci_l1ss_cfg(struct rtw89_dev * rtwdev)4438 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
4439 {
4440 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4441 struct pci_dev *pdev = rtwpci->pdev;
4442 u32 l1ss_cap_ptr, l1ss_ctrl;
4443
4444 if (rtw89_pci_disable_l1ss)
4445 return;
4446
4447 l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
4448 if (!l1ss_cap_ptr)
4449 return;
4450
4451 pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl);
4452
4453 if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
4454 rtw89_pci_l1ss_set(rtwdev, true);
4455 }
4456
rtw89_pci_cpl_timeout_cfg(struct rtw89_dev * rtwdev)4457 static void rtw89_pci_cpl_timeout_cfg(struct rtw89_dev *rtwdev)
4458 {
4459 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4460 struct pci_dev *pdev = rtwpci->pdev;
4461
4462 pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
4463 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS);
4464 }
4465
rtw89_pci_poll_io_idle_ax(struct rtw89_dev * rtwdev)4466 static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
4467 {
4468 int ret = 0;
4469 u32 sts;
4470 u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
4471
4472 ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
4473 10, 1000, false, rtwdev,
4474 R_AX_PCIE_DMA_BUSY1);
4475 if (ret) {
4476 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
4477 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
4478 return -EINVAL;
4479 }
4480 return ret;
4481 }
4482
rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev * rtwdev)4483 static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
4484 {
4485 u32 val;
4486 int ret;
4487
4488 if (rtwdev->chip->chip_id == RTL8852C)
4489 return 0;
4490
4491 rtw89_pci_ctrl_dma_all(rtwdev, false);
4492 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4493 if (ret) {
4494 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4495 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4496 "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
4497 R_AX_DBG_ERR_FLAG, val);
4498 if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
4499 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
4500 if (val & B_AX_RX_STUCK)
4501 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
4502 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4503 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4504 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4505 rtw89_debug(rtwdev, RTW89_DBG_HCI,
4506 "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
4507 R_AX_DBG_ERR_FLAG, val);
4508 }
4509
4510 return ret;
4511 }
4512
rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev * rtwdev)4513 static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
4514 {
4515 int ret;
4516
4517 if (rtwdev->chip->chip_id == RTL8852C)
4518 return 0;
4519
4520 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
4521 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
4522 rtw89_pci_clr_idx_all(rtwdev);
4523
4524 ret = rtw89_pci_rst_bdram_ax(rtwdev);
4525 if (ret)
4526 return ret;
4527
4528 rtw89_pci_ctrl_dma_all(rtwdev, true);
4529 return 0;
4530 }
4531
rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev * rtwdev,enum rtw89_lv1_rcvy_step step)4532 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
4533 enum rtw89_lv1_rcvy_step step)
4534 {
4535 const struct rtw89_pci_info *info = rtwdev->pci_info;
4536 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4537 int ret;
4538
4539 switch (step) {
4540 case RTW89_LV1_RCVY_STEP_1:
4541 ret = gen_def->lv1rst_stop_dma(rtwdev);
4542 if (ret)
4543 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
4544
4545 break;
4546
4547 case RTW89_LV1_RCVY_STEP_2:
4548 ret = gen_def->lv1rst_start_dma(rtwdev);
4549 if (ret)
4550 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
4551 break;
4552
4553 default:
4554 return -EINVAL;
4555 }
4556
4557 return ret;
4558 }
4559
rtw89_pci_ops_dump_err_status(struct rtw89_dev * rtwdev)4560 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
4561 {
4562 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
4563 return;
4564
4565 if (rtwdev->chip->chip_id == RTL8852C) {
4566 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4567 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1));
4568 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4569 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1));
4570 } else {
4571 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
4572 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
4573 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4574 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
4575 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4576 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
4577 }
4578 }
4579
rtw89_pci_napi_poll(struct napi_struct * napi,int budget)4580 static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
4581 {
4582 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
4583 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4584 const struct rtw89_pci_info *info = rtwdev->pci_info;
4585 const struct rtw89_pci_isr_def *isr_def = info->isr_def;
4586 unsigned long flags;
4587 int work_done;
4588
4589 rtwdev->napi_budget_countdown = budget;
4590
4591 rtw89_write32(rtwdev, isr_def->isr_clear_rpq.addr, isr_def->isr_clear_rpq.data);
4592 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4593 if (work_done == budget)
4594 return budget;
4595
4596 rtw89_write32(rtwdev, isr_def->isr_clear_rxq.addr, isr_def->isr_clear_rxq.data);
4597 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
4598 if (work_done < budget && napi_complete_done(napi, work_done)) {
4599 spin_lock_irqsave(&rtwpci->irq_lock, flags);
4600 if (likely(rtwpci->running))
4601 rtw89_chip_enable_intr(rtwdev, rtwpci);
4602 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
4603 }
4604
4605 return work_done;
4606 }
4607
4608 static
rtw89_check_pci_ssid_quirks(struct rtw89_dev * rtwdev,struct pci_dev * pdev,const struct rtw89_pci_ssid_quirk * ssid_quirks)4609 void rtw89_check_pci_ssid_quirks(struct rtw89_dev *rtwdev,
4610 struct pci_dev *pdev,
4611 const struct rtw89_pci_ssid_quirk *ssid_quirks)
4612 {
4613 int i;
4614
4615 if (!ssid_quirks)
4616 return;
4617
4618 for (i = 0; i < 200; i++, ssid_quirks++) {
4619 if (ssid_quirks->vendor == 0 && ssid_quirks->device == 0)
4620 break;
4621
4622 if (ssid_quirks->vendor != pdev->vendor ||
4623 ssid_quirks->device != pdev->device ||
4624 ssid_quirks->subsystem_vendor != pdev->subsystem_vendor ||
4625 ssid_quirks->subsystem_device != pdev->subsystem_device)
4626 continue;
4627
4628 bitmap_or(rtwdev->quirks, rtwdev->quirks, &ssid_quirks->bitmap,
4629 NUM_OF_RTW89_QUIRKS);
4630 rtwdev->custid = ssid_quirks->custid;
4631 break;
4632 }
4633
4634 rtw89_debug(rtwdev, RTW89_DBG_HCI, "quirks=%*ph custid=%d\n",
4635 (int)sizeof(rtwdev->quirks), rtwdev->quirks, rtwdev->custid);
4636 }
4637
rtw89_pci_suspend(struct device * dev)4638 static int __maybe_unused rtw89_pci_suspend(struct device *dev)
4639 {
4640 struct ieee80211_hw *hw = dev_get_drvdata(dev);
4641 struct rtw89_dev *rtwdev = hw->priv;
4642 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4643
4644 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4645 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4646 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4647 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4648 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
4649 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
4650 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
4651 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
4652 } else {
4653 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4654 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
4655 }
4656
4657 return 0;
4658 }
4659
rtw89_pci_l2_hci_ldo(struct rtw89_dev * rtwdev)4660 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
4661 {
4662 if (rtwdev->chip->chip_id == RTL8852C)
4663 return;
4664
4665 /* Hardware need write the reg twice to ensure the setting work */
4666 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4667 RTW89_PCIE_BIT_CFG_RST_MSTATE);
4668 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4669 RTW89_PCIE_BIT_CFG_RST_MSTATE);
4670 }
4671
rtw89_pci_basic_cfg(struct rtw89_dev * rtwdev,bool resume)4672 void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume)
4673 {
4674 if (resume)
4675 rtw89_pci_cfg_dac(rtwdev, false);
4676
4677 rtw89_pci_disable_eq(rtwdev);
4678 rtw89_pci_filter_out(rtwdev);
4679 rtw89_pci_cpl_timeout_cfg(rtwdev);
4680 rtw89_pci_link_cfg(rtwdev);
4681 rtw89_pci_l1ss_cfg(rtwdev);
4682 }
4683
rtw89_pci_resume(struct device * dev)4684 static int __maybe_unused rtw89_pci_resume(struct device *dev)
4685 {
4686 struct ieee80211_hw *hw = dev_get_drvdata(dev);
4687 struct rtw89_dev *rtwdev = hw->priv;
4688 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4689
4690 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4691 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4692 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4693 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4694 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
4695 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
4696 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
4697 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
4698 } else {
4699 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4700 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
4701 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4702 B_AX_SEL_REQ_ENTR_L1);
4703 }
4704 rtw89_pci_hci_ldo(rtwdev);
4705 rtw89_pci_l2_hci_ldo(rtwdev);
4706
4707 rtw89_pci_basic_cfg(rtwdev, true);
4708
4709 return 0;
4710 }
4711
4712 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
4713 EXPORT_SYMBOL(rtw89_pm_ops);
4714
rtw89_pci_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)4715 static pci_ers_result_t rtw89_pci_io_error_detected(struct pci_dev *pdev,
4716 pci_channel_state_t state)
4717 {
4718 struct net_device *netdev = pci_get_drvdata(pdev);
4719
4720 netif_device_detach(netdev);
4721
4722 return PCI_ERS_RESULT_NEED_RESET;
4723 }
4724
rtw89_pci_io_slot_reset(struct pci_dev * pdev)4725 static pci_ers_result_t rtw89_pci_io_slot_reset(struct pci_dev *pdev)
4726 {
4727 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
4728 struct rtw89_dev *rtwdev = hw->priv;
4729
4730 rtw89_ser_notify(rtwdev, MAC_AX_ERR_ASSERTION);
4731
4732 return PCI_ERS_RESULT_RECOVERED;
4733 }
4734
rtw89_pci_io_resume(struct pci_dev * pdev)4735 static void rtw89_pci_io_resume(struct pci_dev *pdev)
4736 {
4737 struct net_device *netdev = pci_get_drvdata(pdev);
4738
4739 /* ack any pending wake events, disable PME */
4740 pci_enable_wake(pdev, PCI_D0, 0);
4741
4742 netif_device_attach(netdev);
4743 }
4744
4745 const struct pci_error_handlers rtw89_pci_err_handler = {
4746 .error_detected = rtw89_pci_io_error_detected,
4747 .slot_reset = rtw89_pci_io_slot_reset,
4748 .resume = rtw89_pci_io_resume,
4749 };
4750 EXPORT_SYMBOL(rtw89_pci_err_handler);
4751
4752 const struct rtw89_pci_isr_def rtw89_pci_isr_ax = {
4753 .isr_rdu = B_AX_RDU_INT,
4754 .isr_halt_c2h = B_AX_HALT_C2H_INT_EN,
4755 .isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN,
4756 .isr_sps_ocp = 0,
4757 .isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT},
4758 .isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT |
4759 B_AX_RDU_INT},
4760 };
4761 EXPORT_SYMBOL(rtw89_pci_isr_ax);
4762
4763 const struct rtw89_pci_gen_def rtw89_pci_gen_ax = {
4764 .mac_pre_init = rtw89_pci_ops_mac_pre_init_ax,
4765 .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit_ax,
4766 .mac_post_init = rtw89_pci_ops_mac_post_init_ax,
4767
4768 .clr_idx_all = rtw89_pci_clr_idx_all_ax,
4769 .rst_bdram = rtw89_pci_rst_bdram_ax,
4770
4771 .lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax,
4772 .lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax,
4773
4774 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_ax,
4775 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_ax,
4776 .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle_ax,
4777
4778 .aspm_set = rtw89_pci_aspm_set_ax,
4779 .clkreq_set = rtw89_pci_clkreq_set_ax,
4780 .l1ss_set = rtw89_pci_l1ss_set_ax,
4781
4782 .disable_eq = rtw89_pci_disable_eq_ax,
4783 .power_wake = rtw89_pci_power_wake_ax,
4784 };
4785 EXPORT_SYMBOL(rtw89_pci_gen_ax);
4786
4787 static const struct rtw89_hci_ops rtw89_pci_ops = {
4788 .tx_write = rtw89_pci_ops_tx_write,
4789 .tx_kick_off = rtw89_pci_ops_tx_kick_off,
4790 .flush_queues = rtw89_pci_ops_flush_queues,
4791 .reset = rtw89_pci_ops_reset,
4792 .start = rtw89_pci_ops_start,
4793 .stop = rtw89_pci_ops_stop,
4794 .pause = rtw89_pci_ops_pause,
4795 .switch_mode = rtw89_pci_ops_switch_mode,
4796 .recalc_int_mit = rtw89_pci_recalc_int_mit,
4797
4798 .read8 = rtw89_pci_ops_read8,
4799 .read16 = rtw89_pci_ops_read16,
4800 .read32 = rtw89_pci_ops_read32,
4801 .write8 = rtw89_pci_ops_write8,
4802 .write16 = rtw89_pci_ops_write16,
4803 .write32 = rtw89_pci_ops_write32,
4804
4805 .read32_pci_cfg = rtw89_pci_ops_read32_pci_cfg,
4806
4807 .mac_pre_init = rtw89_pci_ops_mac_pre_init,
4808 .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit,
4809 .mac_post_init = rtw89_pci_ops_mac_post_init,
4810 .deinit = rtw89_pci_ops_deinit,
4811
4812 .check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
4813 .mac_lv1_rcvy = rtw89_pci_ops_mac_lv1_recovery,
4814 .dump_err_status = rtw89_pci_ops_dump_err_status,
4815 .napi_poll = rtw89_pci_napi_poll,
4816
4817 .recovery_start = rtw89_pci_ops_recovery_start,
4818 .recovery_complete = rtw89_pci_ops_recovery_complete,
4819
4820 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch,
4821 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch,
4822 .ctrl_trxhci = rtw89_pci_ctrl_dma_trx,
4823 .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle,
4824
4825 .clr_idx_all = rtw89_pci_clr_idx_all,
4826 .clear = rtw89_pci_clear_resource,
4827 .disable_intr = rtw89_pci_disable_intr_lock,
4828 .enable_intr = rtw89_pci_enable_intr_lock,
4829 .rst_bdram = rtw89_pci_reset_bdram,
4830 };
4831
rtw89_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)4832 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4833 {
4834 struct rtw89_dev *rtwdev;
4835 const struct rtw89_driver_info *info;
4836 const struct rtw89_pci_info *pci_info;
4837 int ret;
4838
4839 info = (const struct rtw89_driver_info *)id->driver_data;
4840
4841 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
4842 sizeof(struct rtw89_pci),
4843 info->chip, info->variant);
4844 if (!rtwdev) {
4845 dev_err(&pdev->dev, "failed to allocate hw\n");
4846 return -ENOMEM;
4847 }
4848
4849 pci_info = info->bus.pci;
4850
4851 rtwdev->pci_info = info->bus.pci;
4852 rtwdev->hci.ops = &rtw89_pci_ops;
4853 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
4854 rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_PCIE;
4855 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
4856 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
4857
4858 rtw89_check_quirks(rtwdev, info->quirks);
4859 rtw89_check_pci_ssid_quirks(rtwdev, pdev, pci_info->ssid_quirks);
4860
4861 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
4862
4863 ret = rtw89_core_init(rtwdev);
4864 if (ret) {
4865 rtw89_err(rtwdev, "failed to initialise core\n");
4866 goto err_release_hw;
4867 }
4868
4869 ret = rtw89_pci_claim_device(rtwdev, pdev);
4870 if (ret) {
4871 rtw89_err(rtwdev, "failed to claim pci device\n");
4872 goto err_core_deinit;
4873 }
4874
4875 ret = rtw89_pci_setup_resource(rtwdev, pdev);
4876 if (ret) {
4877 rtw89_err(rtwdev, "failed to setup pci resource\n");
4878 goto err_declaim_pci;
4879 }
4880
4881 ret = rtw89_chip_info_setup(rtwdev);
4882 if (ret) {
4883 rtw89_err(rtwdev, "failed to setup chip information\n");
4884 goto err_clear_resource;
4885 }
4886
4887 rtw89_pci_basic_cfg(rtwdev, false);
4888
4889 ret = rtw89_core_napi_init(rtwdev);
4890 if (ret) {
4891 rtw89_err(rtwdev, "failed to init napi\n");
4892 goto err_clear_resource;
4893 }
4894
4895 ret = rtw89_pci_request_irq(rtwdev, pdev);
4896 if (ret) {
4897 rtw89_err(rtwdev, "failed to request pci irq\n");
4898 goto err_deinit_napi;
4899 }
4900
4901 ret = rtw89_core_register(rtwdev);
4902 if (ret) {
4903 rtw89_err(rtwdev, "failed to register core\n");
4904 goto err_free_irq;
4905 }
4906
4907 set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags);
4908
4909 return 0;
4910
4911 err_free_irq:
4912 rtw89_pci_free_irq(rtwdev, pdev);
4913 err_deinit_napi:
4914 rtw89_core_napi_deinit(rtwdev);
4915 err_clear_resource:
4916 rtw89_pci_clear_resource(rtwdev, pdev);
4917 err_declaim_pci:
4918 rtw89_pci_declaim_device(rtwdev, pdev);
4919 err_core_deinit:
4920 rtw89_core_deinit(rtwdev);
4921 err_release_hw:
4922 rtw89_free_ieee80211_hw(rtwdev);
4923
4924 return ret;
4925 }
4926 EXPORT_SYMBOL(rtw89_pci_probe);
4927
rtw89_pci_remove(struct pci_dev * pdev)4928 void rtw89_pci_remove(struct pci_dev *pdev)
4929 {
4930 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
4931 struct rtw89_dev *rtwdev;
4932
4933 rtwdev = hw->priv;
4934
4935 rtw89_pci_free_irq(rtwdev, pdev);
4936 rtw89_core_napi_deinit(rtwdev);
4937 rtw89_core_unregister(rtwdev);
4938 rtw89_pci_clear_resource(rtwdev, pdev);
4939 rtw89_pci_declaim_device(rtwdev, pdev);
4940 rtw89_core_deinit(rtwdev);
4941 rtw89_free_ieee80211_hw(rtwdev);
4942 }
4943 EXPORT_SYMBOL(rtw89_pci_remove);
4944
4945 MODULE_AUTHOR("Realtek Corporation");
4946 MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver");
4947 MODULE_LICENSE("Dual BSD/GPL");
4948