1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright(c) 2018-2019 Realtek Corporation 3 */ 4 5 #if defined(__FreeBSD__) 6 #define LINUXKPI_PARAM_PREFIX rtw88_pci_ 7 #endif 8 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 #include "main.h" 12 #include "pci.h" 13 #include "reg.h" 14 #include "tx.h" 15 #include "rx.h" 16 #include "fw.h" 17 #include "ps.h" 18 #include "debug.h" 19 #if defined(__FreeBSD__) 20 #include <sys/rman.h> 21 #include <linux/pm.h> 22 #endif 23 24 static bool rtw_disable_msi; 25 static bool rtw_pci_disable_aspm; 26 module_param_named(disable_msi, rtw_disable_msi, bool, 0644); 27 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644); 28 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support"); 29 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support"); 30 31 static u32 rtw_pci_tx_queue_idx_addr[] = { 32 [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ, 33 [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ, 34 [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ, 35 [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ, 36 [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ, 37 [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q, 38 [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ, 39 }; 40 41 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, 42 enum rtw_tx_queue_type queue) 43 { 44 switch (queue) { 45 case RTW_TX_QUEUE_BCN: 46 return TX_DESC_QSEL_BEACON; 47 case RTW_TX_QUEUE_H2C: 48 return TX_DESC_QSEL_H2C; 49 case RTW_TX_QUEUE_MGMT: 50 return TX_DESC_QSEL_MGMT; 51 case RTW_TX_QUEUE_HI0: 52 return TX_DESC_QSEL_HIGH; 53 default: 54 return skb->priority; 55 } 56 }; 57 58 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr) 59 { 60 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 61 62 #if defined(__linux__) 63 return readb(rtwpci->mmap + addr); 64 #elif defined(__FreeBSD__) 65 u8 val; 66 67 val = bus_read_1((struct resource *)rtwpci->mmap, addr); 68 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val); 69 return (val); 70 #endif 71 } 72 73 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr) 74 { 75 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 76 77 #if defined(__linux__) 78 return readw(rtwpci->mmap + addr); 79 #elif defined(__FreeBSD__) 80 u16 val; 81 82 val = bus_read_2((struct resource *)rtwpci->mmap, addr); 83 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val); 84 return (val); 85 #endif 86 } 87 88 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr) 89 { 90 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 91 92 #if defined(__linux__) 93 return readl(rtwpci->mmap + addr); 94 #elif defined(__FreeBSD__) 95 u32 val; 96 97 val = bus_read_4((struct resource *)rtwpci->mmap, addr); 98 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val); 99 return (val); 100 #endif 101 } 102 103 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val) 104 { 105 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 106 107 #if defined(__linux__) 108 writeb(val, rtwpci->mmap + addr); 109 #elif defined(__FreeBSD__) 110 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, val); 111 return (bus_write_1((struct resource *)rtwpci->mmap, addr, val)); 112 #endif 113 } 114 115 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val) 116 { 117 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 118 119 #if defined(__linux__) 120 writew(val, rtwpci->mmap + addr); 121 #elif defined(__FreeBSD__) 122 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, val); 123 return (bus_write_2((struct resource *)rtwpci->mmap, addr, val)); 124 #endif 125 } 126 127 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val) 128 { 129 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 130 131 #if defined(__linux__) 132 writel(val, rtwpci->mmap + addr); 133 #elif defined(__FreeBSD__) 134 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, val); 135 return (bus_write_4((struct resource *)rtwpci->mmap, addr, val)); 136 #endif 137 } 138 139 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev, 140 struct rtw_pci_tx_ring *tx_ring) 141 { 142 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 143 struct rtw_pci_tx_data *tx_data; 144 struct sk_buff *skb, *tmp; 145 dma_addr_t dma; 146 147 /* free every skb remained in tx list */ 148 skb_queue_walk_safe(&tx_ring->queue, skb, tmp) { 149 __skb_unlink(skb, &tx_ring->queue); 150 tx_data = rtw_pci_get_tx_data(skb); 151 dma = tx_data->dma; 152 153 dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE); 154 dev_kfree_skb_any(skb); 155 } 156 } 157 158 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev, 159 struct rtw_pci_tx_ring *tx_ring) 160 { 161 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 162 u8 *head = tx_ring->r.head; 163 u32 len = tx_ring->r.len; 164 int ring_sz = len * tx_ring->r.desc_size; 165 166 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 167 168 /* free the ring itself */ 169 dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma); 170 tx_ring->r.head = NULL; 171 } 172 173 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev, 174 struct rtw_pci_rx_ring *rx_ring) 175 { 176 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 177 struct sk_buff *skb; 178 int buf_sz = RTK_PCI_RX_BUF_SIZE; 179 dma_addr_t dma; 180 int i; 181 182 for (i = 0; i < rx_ring->r.len; i++) { 183 skb = rx_ring->buf[i]; 184 if (!skb) 185 continue; 186 187 dma = *((dma_addr_t *)skb->cb); 188 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 189 dev_kfree_skb(skb); 190 rx_ring->buf[i] = NULL; 191 } 192 } 193 194 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev, 195 struct rtw_pci_rx_ring *rx_ring) 196 { 197 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 198 u8 *head = rx_ring->r.head; 199 int ring_sz = rx_ring->r.desc_size * rx_ring->r.len; 200 201 rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring); 202 203 dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma); 204 } 205 206 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev) 207 { 208 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 209 struct rtw_pci_tx_ring *tx_ring; 210 struct rtw_pci_rx_ring *rx_ring; 211 int i; 212 213 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 214 tx_ring = &rtwpci->tx_rings[i]; 215 rtw_pci_free_tx_ring(rtwdev, tx_ring); 216 } 217 218 for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) { 219 rx_ring = &rtwpci->rx_rings[i]; 220 rtw_pci_free_rx_ring(rtwdev, rx_ring); 221 } 222 } 223 224 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev, 225 struct rtw_pci_tx_ring *tx_ring, 226 u8 desc_size, u32 len) 227 { 228 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 229 int ring_sz = desc_size * len; 230 dma_addr_t dma; 231 u8 *head; 232 233 if (len > TRX_BD_IDX_MASK) { 234 rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len); 235 return -EINVAL; 236 } 237 238 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 239 if (!head) { 240 rtw_err(rtwdev, "failed to allocate tx ring\n"); 241 return -ENOMEM; 242 } 243 244 skb_queue_head_init(&tx_ring->queue); 245 tx_ring->r.head = head; 246 tx_ring->r.dma = dma; 247 tx_ring->r.len = len; 248 tx_ring->r.desc_size = desc_size; 249 tx_ring->r.wp = 0; 250 tx_ring->r.rp = 0; 251 252 return 0; 253 } 254 255 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb, 256 struct rtw_pci_rx_ring *rx_ring, 257 u32 idx, u32 desc_sz) 258 { 259 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 260 struct rtw_pci_rx_buffer_desc *buf_desc; 261 int buf_sz = RTK_PCI_RX_BUF_SIZE; 262 dma_addr_t dma; 263 264 if (!skb) 265 return -EINVAL; 266 267 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); 268 if (dma_mapping_error(&pdev->dev, dma)) 269 return -EBUSY; 270 271 *((dma_addr_t *)skb->cb) = dma; 272 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 273 idx * desc_sz); 274 memset(buf_desc, 0, sizeof(*buf_desc)); 275 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 276 buf_desc->dma = cpu_to_le32(dma); 277 278 return 0; 279 } 280 281 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma, 282 struct rtw_pci_rx_ring *rx_ring, 283 u32 idx, u32 desc_sz) 284 { 285 struct device *dev = rtwdev->dev; 286 struct rtw_pci_rx_buffer_desc *buf_desc; 287 int buf_sz = RTK_PCI_RX_BUF_SIZE; 288 289 dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE); 290 291 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 292 idx * desc_sz); 293 memset(buf_desc, 0, sizeof(*buf_desc)); 294 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 295 buf_desc->dma = cpu_to_le32(dma); 296 } 297 298 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev, 299 struct rtw_pci_rx_ring *rx_ring, 300 u8 desc_size, u32 len) 301 { 302 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 303 struct sk_buff *skb = NULL; 304 dma_addr_t dma; 305 u8 *head; 306 int ring_sz = desc_size * len; 307 int buf_sz = RTK_PCI_RX_BUF_SIZE; 308 int i, allocated; 309 int ret = 0; 310 311 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 312 if (!head) { 313 rtw_err(rtwdev, "failed to allocate rx ring\n"); 314 return -ENOMEM; 315 } 316 rx_ring->r.head = head; 317 318 for (i = 0; i < len; i++) { 319 skb = dev_alloc_skb(buf_sz); 320 if (!skb) { 321 allocated = i; 322 ret = -ENOMEM; 323 goto err_out; 324 } 325 326 memset(skb->data, 0, buf_sz); 327 rx_ring->buf[i] = skb; 328 ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size); 329 if (ret) { 330 allocated = i; 331 dev_kfree_skb_any(skb); 332 goto err_out; 333 } 334 } 335 336 rx_ring->r.dma = dma; 337 rx_ring->r.len = len; 338 rx_ring->r.desc_size = desc_size; 339 rx_ring->r.wp = 0; 340 rx_ring->r.rp = 0; 341 342 return 0; 343 344 err_out: 345 for (i = 0; i < allocated; i++) { 346 skb = rx_ring->buf[i]; 347 if (!skb) 348 continue; 349 dma = *((dma_addr_t *)skb->cb); 350 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 351 dev_kfree_skb_any(skb); 352 rx_ring->buf[i] = NULL; 353 } 354 dma_free_coherent(&pdev->dev, ring_sz, head, dma); 355 356 rtw_err(rtwdev, "failed to init rx buffer\n"); 357 358 return ret; 359 } 360 361 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev) 362 { 363 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 364 struct rtw_pci_tx_ring *tx_ring; 365 struct rtw_pci_rx_ring *rx_ring; 366 const struct rtw_chip_info *chip = rtwdev->chip; 367 int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0; 368 int tx_desc_size, rx_desc_size; 369 u32 len; 370 int ret; 371 372 tx_desc_size = chip->tx_buf_desc_sz; 373 374 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 375 tx_ring = &rtwpci->tx_rings[i]; 376 len = max_num_of_tx_queue(i); 377 ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len); 378 if (ret) 379 goto out; 380 } 381 382 rx_desc_size = chip->rx_buf_desc_sz; 383 384 for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) { 385 rx_ring = &rtwpci->rx_rings[j]; 386 ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size, 387 RTK_MAX_RX_DESC_NUM); 388 if (ret) 389 goto out; 390 } 391 392 return 0; 393 394 out: 395 tx_alloced = i; 396 for (i = 0; i < tx_alloced; i++) { 397 tx_ring = &rtwpci->tx_rings[i]; 398 rtw_pci_free_tx_ring(rtwdev, tx_ring); 399 } 400 401 rx_alloced = j; 402 for (j = 0; j < rx_alloced; j++) { 403 rx_ring = &rtwpci->rx_rings[j]; 404 rtw_pci_free_rx_ring(rtwdev, rx_ring); 405 } 406 407 return ret; 408 } 409 410 static void rtw_pci_deinit(struct rtw_dev *rtwdev) 411 { 412 rtw_pci_free_trx_ring(rtwdev); 413 } 414 415 static int rtw_pci_init(struct rtw_dev *rtwdev) 416 { 417 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 418 int ret = 0; 419 420 rtwpci->irq_mask[0] = IMR_HIGHDOK | 421 IMR_MGNTDOK | 422 IMR_BKDOK | 423 IMR_BEDOK | 424 IMR_VIDOK | 425 IMR_VODOK | 426 IMR_ROK | 427 IMR_BCNDMAINT_E | 428 IMR_C2HCMD | 429 0; 430 rtwpci->irq_mask[1] = IMR_TXFOVW | 431 0; 432 rtwpci->irq_mask[3] = IMR_H2CDOK | 433 0; 434 spin_lock_init(&rtwpci->irq_lock); 435 spin_lock_init(&rtwpci->hwirq_lock); 436 ret = rtw_pci_init_trx_ring(rtwdev); 437 438 return ret; 439 } 440 441 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev) 442 { 443 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 444 u32 len; 445 u8 tmp; 446 dma_addr_t dma; 447 448 tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3); 449 rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7); 450 451 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma; 452 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma); 453 454 if (!rtw_chip_wcpu_11n(rtwdev)) { 455 len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len; 456 dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma; 457 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0; 458 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0; 459 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK); 460 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma); 461 } 462 463 len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len; 464 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma; 465 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0; 466 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0; 467 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK); 468 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma); 469 470 len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len; 471 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma; 472 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0; 473 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0; 474 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK); 475 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma); 476 477 len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len; 478 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma; 479 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0; 480 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0; 481 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK); 482 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma); 483 484 len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len; 485 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma; 486 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0; 487 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0; 488 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK); 489 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma); 490 491 len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len; 492 dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma; 493 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0; 494 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0; 495 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK); 496 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma); 497 498 len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len; 499 dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma; 500 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0; 501 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0; 502 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK); 503 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma); 504 505 len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len; 506 dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma; 507 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0; 508 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0; 509 rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK); 510 rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma); 511 512 /* reset read/write point */ 513 rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff); 514 515 /* reset H2C Queue index in a single write */ 516 if (rtw_chip_wcpu_11ac(rtwdev)) 517 rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR, 518 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX); 519 } 520 521 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev) 522 { 523 rtw_pci_reset_buf_desc(rtwdev); 524 } 525 526 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev, 527 struct rtw_pci *rtwpci, bool exclude_rx) 528 { 529 unsigned long flags; 530 u32 imr0_unmask = exclude_rx ? IMR_ROK : 0; 531 532 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 533 534 rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask); 535 rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]); 536 if (rtw_chip_wcpu_11ac(rtwdev)) 537 rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]); 538 539 rtwpci->irq_enabled = true; 540 541 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 542 } 543 544 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev, 545 struct rtw_pci *rtwpci) 546 { 547 unsigned long flags; 548 549 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 550 551 if (!rtwpci->irq_enabled) 552 goto out; 553 554 rtw_write32(rtwdev, RTK_PCI_HIMR0, 0); 555 rtw_write32(rtwdev, RTK_PCI_HIMR1, 0); 556 if (rtw_chip_wcpu_11ac(rtwdev)) 557 rtw_write32(rtwdev, RTK_PCI_HIMR3, 0); 558 559 rtwpci->irq_enabled = false; 560 561 out: 562 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 563 } 564 565 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 566 { 567 /* reset dma and rx tag */ 568 rtw_write32_set(rtwdev, RTK_PCI_CTRL, 569 BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN); 570 rtwpci->rx_tag = 0; 571 } 572 573 static int rtw_pci_setup(struct rtw_dev *rtwdev) 574 { 575 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 576 577 rtw_pci_reset_trx_ring(rtwdev); 578 rtw_pci_dma_reset(rtwdev, rtwpci); 579 580 return 0; 581 } 582 583 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 584 { 585 struct rtw_pci_tx_ring *tx_ring; 586 enum rtw_tx_queue_type queue; 587 588 rtw_pci_reset_trx_ring(rtwdev); 589 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 590 tx_ring = &rtwpci->tx_rings[queue]; 591 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 592 } 593 } 594 595 static void rtw_pci_napi_start(struct rtw_dev *rtwdev) 596 { 597 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 598 599 if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 600 return; 601 602 napi_enable(&rtwpci->napi); 603 } 604 605 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev) 606 { 607 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 608 609 if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 610 return; 611 612 napi_synchronize(&rtwpci->napi); 613 napi_disable(&rtwpci->napi); 614 } 615 616 static int rtw_pci_start(struct rtw_dev *rtwdev) 617 { 618 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 619 620 rtw_pci_napi_start(rtwdev); 621 622 spin_lock_bh(&rtwpci->irq_lock); 623 rtwpci->running = true; 624 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 625 spin_unlock_bh(&rtwpci->irq_lock); 626 627 return 0; 628 } 629 630 static void rtw_pci_stop(struct rtw_dev *rtwdev) 631 { 632 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 633 struct pci_dev *pdev = rtwpci->pdev; 634 635 spin_lock_bh(&rtwpci->irq_lock); 636 rtwpci->running = false; 637 rtw_pci_disable_interrupt(rtwdev, rtwpci); 638 spin_unlock_bh(&rtwpci->irq_lock); 639 640 synchronize_irq(pdev->irq); 641 rtw_pci_napi_stop(rtwdev); 642 643 spin_lock_bh(&rtwpci->irq_lock); 644 rtw_pci_dma_release(rtwdev, rtwpci); 645 spin_unlock_bh(&rtwpci->irq_lock); 646 } 647 648 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev) 649 { 650 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 651 struct rtw_pci_tx_ring *tx_ring; 652 enum rtw_tx_queue_type queue; 653 bool tx_empty = true; 654 655 if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 656 goto enter_deep_ps; 657 658 lockdep_assert_held(&rtwpci->irq_lock); 659 660 /* Deep PS state is not allowed to TX-DMA */ 661 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 662 /* BCN queue is rsvd page, does not have DMA interrupt 663 * H2C queue is managed by firmware 664 */ 665 if (queue == RTW_TX_QUEUE_BCN || 666 queue == RTW_TX_QUEUE_H2C) 667 continue; 668 669 tx_ring = &rtwpci->tx_rings[queue]; 670 671 /* check if there is any skb DMAing */ 672 if (skb_queue_len(&tx_ring->queue)) { 673 tx_empty = false; 674 break; 675 } 676 } 677 678 if (!tx_empty) { 679 rtw_dbg(rtwdev, RTW_DBG_PS, 680 "TX path not empty, cannot enter deep power save state\n"); 681 return; 682 } 683 enter_deep_ps: 684 set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags); 685 rtw_power_mode_change(rtwdev, true); 686 } 687 688 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev) 689 { 690 #if defined(__linux__) 691 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 692 693 lockdep_assert_held(&rtwpci->irq_lock); 694 #elif defined(__FreeBSD__) 695 lockdep_assert_held(&((struct rtw_pci *)rtwdev->priv)->irq_lock); 696 #endif 697 698 if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 699 rtw_power_mode_change(rtwdev, false); 700 } 701 702 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter) 703 { 704 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 705 706 spin_lock_bh(&rtwpci->irq_lock); 707 708 if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 709 rtw_pci_deep_ps_enter(rtwdev); 710 711 if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 712 rtw_pci_deep_ps_leave(rtwdev); 713 714 spin_unlock_bh(&rtwpci->irq_lock); 715 } 716 717 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci, 718 struct rtw_pci_tx_ring *ring) 719 { 720 struct sk_buff *prev = skb_dequeue(&ring->queue); 721 struct rtw_pci_tx_data *tx_data; 722 dma_addr_t dma; 723 724 if (!prev) 725 return; 726 727 tx_data = rtw_pci_get_tx_data(prev); 728 dma = tx_data->dma; 729 dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE); 730 dev_kfree_skb_any(prev); 731 } 732 733 static void rtw_pci_dma_check(struct rtw_dev *rtwdev, 734 struct rtw_pci_rx_ring *rx_ring, 735 u32 idx) 736 { 737 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 738 const struct rtw_chip_info *chip = rtwdev->chip; 739 struct rtw_pci_rx_buffer_desc *buf_desc; 740 u32 desc_sz = chip->rx_buf_desc_sz; 741 u16 total_pkt_size; 742 743 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 744 idx * desc_sz); 745 total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size); 746 747 /* rx tag mismatch, throw a warning */ 748 if (total_pkt_size != rtwpci->rx_tag) 749 rtw_warn(rtwdev, "pci bus timeout, check dma status\n"); 750 751 rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX; 752 } 753 754 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q) 755 { 756 u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q]; 757 u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2); 758 759 return FIELD_GET(TRX_BD_IDX_MASK, bd_idx); 760 } 761 762 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop) 763 { 764 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 765 struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q]; 766 u32 cur_rp; 767 u8 i; 768 769 /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a 770 * bit dynamic, it's hard to define a reasonable fixed total timeout to 771 * use read_poll_timeout* helper. Instead, we can ensure a reasonable 772 * polling times, so we just use for loop with udelay here. 773 */ 774 for (i = 0; i < 30; i++) { 775 cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q); 776 if (cur_rp == ring->r.wp) 777 return; 778 779 udelay(1); 780 } 781 782 if (!drop) 783 rtw_dbg(rtwdev, RTW_DBG_UNEXP, 784 "timed out to flush pci tx ring[%d]\n", pci_q); 785 } 786 787 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues, 788 bool drop) 789 { 790 u8 q; 791 792 for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) { 793 /* Unnecessary to flush BCN, H2C and HI tx queues. */ 794 if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C || 795 q == RTW_TX_QUEUE_HI0) 796 continue; 797 798 if (pci_queues & BIT(q)) 799 __pci_flush_queue(rtwdev, q, drop); 800 } 801 } 802 803 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop) 804 { 805 u32 pci_queues = 0; 806 u8 i; 807 808 /* If all of the hardware queues are requested to flush, 809 * flush all of the pci queues. 810 */ 811 if (queues == BIT(rtwdev->hw->queues) - 1) { 812 pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1; 813 } else { 814 for (i = 0; i < rtwdev->hw->queues; i++) 815 if (queues & BIT(i)) 816 pci_queues |= BIT(rtw_tx_ac_to_hwq(i)); 817 } 818 819 __rtw_pci_flush_queues(rtwdev, pci_queues, drop); 820 } 821 822 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, 823 enum rtw_tx_queue_type queue) 824 { 825 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 826 struct rtw_pci_tx_ring *ring; 827 u32 bd_idx; 828 829 ring = &rtwpci->tx_rings[queue]; 830 bd_idx = rtw_pci_tx_queue_idx_addr[queue]; 831 832 spin_lock_bh(&rtwpci->irq_lock); 833 if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 834 rtw_pci_deep_ps_leave(rtwdev); 835 rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK); 836 spin_unlock_bh(&rtwpci->irq_lock); 837 } 838 839 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev) 840 { 841 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 842 enum rtw_tx_queue_type queue; 843 844 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) 845 if (test_and_clear_bit(queue, rtwpci->tx_queued)) 846 rtw_pci_tx_kick_off_queue(rtwdev, queue); 847 } 848 849 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev, 850 struct rtw_tx_pkt_info *pkt_info, 851 struct sk_buff *skb, 852 enum rtw_tx_queue_type queue) 853 { 854 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 855 const struct rtw_chip_info *chip = rtwdev->chip; 856 struct rtw_pci_tx_ring *ring; 857 struct rtw_pci_tx_data *tx_data; 858 dma_addr_t dma; 859 u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz; 860 u32 tx_buf_desc_sz = chip->tx_buf_desc_sz; 861 u32 size; 862 u32 psb_len; 863 u8 *pkt_desc; 864 struct rtw_pci_tx_buffer_desc *buf_desc; 865 866 ring = &rtwpci->tx_rings[queue]; 867 868 size = skb->len; 869 870 if (queue == RTW_TX_QUEUE_BCN) 871 rtw_pci_release_rsvd_page(rtwpci, ring); 872 else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len)) 873 return -ENOSPC; 874 875 pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); 876 memset(pkt_desc, 0, tx_pkt_desc_sz); 877 pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue); 878 rtw_tx_fill_tx_desc(rtwdev, pkt_info, skb); 879 dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len, 880 DMA_TO_DEVICE); 881 if (dma_mapping_error(&rtwpci->pdev->dev, dma)) 882 return -EBUSY; 883 884 /* after this we got dma mapped, there is no way back */ 885 buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz); 886 memset(buf_desc, 0, tx_buf_desc_sz); 887 psb_len = (skb->len - 1) / 128 + 1; 888 if (queue == RTW_TX_QUEUE_BCN) 889 psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET; 890 891 buf_desc[0].psb_len = cpu_to_le16(psb_len); 892 buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz); 893 buf_desc[0].dma = cpu_to_le32(dma); 894 buf_desc[1].buf_size = cpu_to_le16(size); 895 buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz); 896 897 tx_data = rtw_pci_get_tx_data(skb); 898 tx_data->dma = dma; 899 tx_data->sn = pkt_info->sn; 900 901 spin_lock_bh(&rtwpci->irq_lock); 902 903 skb_queue_tail(&ring->queue, skb); 904 905 if (queue == RTW_TX_QUEUE_BCN) 906 goto out_unlock; 907 908 /* update write-index, and kick it off later */ 909 set_bit(queue, rtwpci->tx_queued); 910 if (++ring->r.wp >= ring->r.len) 911 ring->r.wp = 0; 912 913 out_unlock: 914 spin_unlock_bh(&rtwpci->irq_lock); 915 916 return 0; 917 } 918 919 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, 920 u32 size) 921 { 922 struct sk_buff *skb; 923 struct rtw_tx_pkt_info pkt_info = {0}; 924 u8 reg_bcn_work; 925 int ret; 926 927 skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size); 928 if (!skb) 929 return -ENOMEM; 930 931 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN); 932 if (ret) { 933 #if defined(__FreeBSD__) 934 dev_kfree_skb_any(skb); 935 #endif 936 rtw_err(rtwdev, "failed to write rsvd page data\n"); 937 return ret; 938 } 939 940 /* reserved pages go through beacon queue */ 941 reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK); 942 reg_bcn_work |= BIT_PCI_BCNQ_FLAG; 943 rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work); 944 945 return 0; 946 } 947 948 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) 949 { 950 struct sk_buff *skb; 951 struct rtw_tx_pkt_info pkt_info = {0}; 952 int ret; 953 954 skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size); 955 if (!skb) 956 return -ENOMEM; 957 958 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C); 959 if (ret) { 960 #if defined(__FreeBSD__) 961 dev_kfree_skb_any(skb); 962 #endif 963 rtw_err(rtwdev, "failed to write h2c data\n"); 964 return ret; 965 } 966 967 rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C); 968 969 return 0; 970 } 971 972 static int rtw_pci_tx_write(struct rtw_dev *rtwdev, 973 struct rtw_tx_pkt_info *pkt_info, 974 struct sk_buff *skb) 975 { 976 enum rtw_tx_queue_type queue = rtw_tx_queue_mapping(skb); 977 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 978 struct rtw_pci_tx_ring *ring; 979 int ret; 980 981 ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue); 982 if (ret) 983 return ret; 984 985 ring = &rtwpci->tx_rings[queue]; 986 spin_lock_bh(&rtwpci->irq_lock); 987 if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) { 988 ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb)); 989 ring->queue_stopped = true; 990 } 991 spin_unlock_bh(&rtwpci->irq_lock); 992 993 return 0; 994 } 995 996 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 997 u8 hw_queue) 998 { 999 struct ieee80211_hw *hw = rtwdev->hw; 1000 struct ieee80211_tx_info *info; 1001 struct rtw_pci_tx_ring *ring; 1002 struct rtw_pci_tx_data *tx_data; 1003 struct sk_buff *skb; 1004 u32 count; 1005 u32 bd_idx_addr; 1006 u32 bd_idx, cur_rp, rp_idx; 1007 u16 q_map; 1008 1009 ring = &rtwpci->tx_rings[hw_queue]; 1010 1011 bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue]; 1012 bd_idx = rtw_read32(rtwdev, bd_idx_addr); 1013 cur_rp = bd_idx >> 16; 1014 cur_rp &= TRX_BD_IDX_MASK; 1015 rp_idx = ring->r.rp; 1016 if (cur_rp >= ring->r.rp) 1017 count = cur_rp - ring->r.rp; 1018 else 1019 count = ring->r.len - (ring->r.rp - cur_rp); 1020 1021 while (count--) { 1022 skb = skb_dequeue(&ring->queue); 1023 if (!skb) { 1024 rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n", 1025 count, hw_queue, bd_idx, ring->r.rp, cur_rp); 1026 break; 1027 } 1028 tx_data = rtw_pci_get_tx_data(skb); 1029 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, 1030 DMA_TO_DEVICE); 1031 1032 /* just free command packets from host to card */ 1033 if (hw_queue == RTW_TX_QUEUE_H2C) { 1034 dev_kfree_skb_irq(skb); 1035 continue; 1036 } 1037 1038 if (ring->queue_stopped && 1039 avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) { 1040 q_map = skb_get_queue_mapping(skb); 1041 ieee80211_wake_queue(hw, q_map); 1042 ring->queue_stopped = false; 1043 } 1044 1045 if (++rp_idx >= ring->r.len) 1046 rp_idx = 0; 1047 1048 skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz); 1049 1050 info = IEEE80211_SKB_CB(skb); 1051 1052 /* enqueue to wait for tx report */ 1053 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { 1054 rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); 1055 continue; 1056 } 1057 1058 /* always ACK for others, then they won't be marked as drop */ 1059 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 1060 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; 1061 else 1062 info->flags |= IEEE80211_TX_STAT_ACK; 1063 1064 ieee80211_tx_info_clear_status(info); 1065 ieee80211_tx_status_irqsafe(hw, skb); 1066 } 1067 1068 ring->r.rp = cur_rp; 1069 } 1070 1071 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev) 1072 { 1073 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1074 struct napi_struct *napi = &rtwpci->napi; 1075 1076 napi_schedule(napi); 1077 } 1078 1079 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev, 1080 struct rtw_pci *rtwpci) 1081 { 1082 struct rtw_pci_rx_ring *ring; 1083 int count = 0; 1084 u32 tmp, cur_wp; 1085 1086 ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1087 tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ); 1088 cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK); 1089 if (cur_wp >= ring->r.wp) 1090 count = cur_wp - ring->r.wp; 1091 else 1092 count = ring->r.len - (ring->r.wp - cur_wp); 1093 1094 return count; 1095 } 1096 1097 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 1098 u8 hw_queue, u32 limit) 1099 { 1100 const struct rtw_chip_info *chip = rtwdev->chip; 1101 struct napi_struct *napi = &rtwpci->napi; 1102 struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1103 struct rtw_rx_pkt_stat pkt_stat; 1104 struct ieee80211_rx_status rx_status; 1105 struct sk_buff *skb, *new; 1106 u32 cur_rp = ring->r.rp; 1107 u32 count, rx_done = 0; 1108 u32 pkt_offset; 1109 u32 pkt_desc_sz = chip->rx_pkt_desc_sz; 1110 u32 buf_desc_sz = chip->rx_buf_desc_sz; 1111 u32 new_len; 1112 u8 *rx_desc; 1113 dma_addr_t dma; 1114 1115 count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci); 1116 count = min(count, limit); 1117 1118 while (count--) { 1119 rtw_pci_dma_check(rtwdev, ring, cur_rp); 1120 skb = ring->buf[cur_rp]; 1121 dma = *((dma_addr_t *)skb->cb); 1122 dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE, 1123 DMA_FROM_DEVICE); 1124 rx_desc = skb->data; 1125 rtw_rx_query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status); 1126 1127 /* offset from rx_desc to payload */ 1128 pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + 1129 pkt_stat.shift; 1130 1131 /* allocate a new skb for this frame, 1132 * discard the frame if none available 1133 */ 1134 new_len = pkt_stat.pkt_len + pkt_offset; 1135 new = dev_alloc_skb(new_len); 1136 if (WARN_ONCE(!new, "rx routine starvation\n")) 1137 goto next_rp; 1138 1139 /* put the DMA data including rx_desc from phy to new skb */ 1140 skb_put_data(new, skb->data, new_len); 1141 1142 if (pkt_stat.is_c2h) { 1143 rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new); 1144 } else { 1145 /* remove rx_desc */ 1146 skb_pull(new, pkt_offset); 1147 1148 rtw_update_rx_freq_for_invalid(rtwdev, new, &rx_status, &pkt_stat); 1149 rtw_rx_stats(rtwdev, pkt_stat.vif, new); 1150 memcpy(new->cb, &rx_status, sizeof(rx_status)); 1151 ieee80211_rx_napi(rtwdev->hw, NULL, new, napi); 1152 rx_done++; 1153 } 1154 1155 next_rp: 1156 /* new skb delivered to mac80211, re-enable original skb DMA */ 1157 rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp, 1158 buf_desc_sz); 1159 1160 /* host read next element in ring */ 1161 if (++cur_rp >= ring->r.len) 1162 cur_rp = 0; 1163 } 1164 1165 ring->r.rp = cur_rp; 1166 /* 'rp', the last position we have read, is seen as previous posistion 1167 * of 'wp' that is used to calculate 'count' next time. 1168 */ 1169 ring->r.wp = cur_rp; 1170 rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp); 1171 1172 return rx_done; 1173 } 1174 1175 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev, 1176 struct rtw_pci *rtwpci, u32 *irq_status) 1177 { 1178 unsigned long flags; 1179 1180 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 1181 1182 irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0); 1183 irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1); 1184 if (rtw_chip_wcpu_11ac(rtwdev)) 1185 irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3); 1186 else 1187 irq_status[3] = 0; 1188 irq_status[0] &= rtwpci->irq_mask[0]; 1189 irq_status[1] &= rtwpci->irq_mask[1]; 1190 irq_status[3] &= rtwpci->irq_mask[3]; 1191 rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]); 1192 rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]); 1193 if (rtw_chip_wcpu_11ac(rtwdev)) 1194 rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]); 1195 1196 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 1197 } 1198 1199 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev) 1200 { 1201 struct rtw_dev *rtwdev = dev; 1202 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1203 1204 /* disable RTW PCI interrupt to avoid more interrupts before the end of 1205 * thread function 1206 * 1207 * disable HIMR here to also avoid new HISR flag being raised before 1208 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs 1209 * are cleared, the edge-triggered interrupt will not be generated when 1210 * a new HISR flag is set. 1211 */ 1212 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1213 1214 return IRQ_WAKE_THREAD; 1215 } 1216 1217 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev) 1218 { 1219 struct rtw_dev *rtwdev = dev; 1220 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1221 u32 irq_status[4]; 1222 bool rx = false; 1223 1224 spin_lock_bh(&rtwpci->irq_lock); 1225 rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status); 1226 1227 if (irq_status[0] & IMR_MGNTDOK) 1228 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT); 1229 if (irq_status[0] & IMR_HIGHDOK) 1230 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0); 1231 if (irq_status[0] & IMR_BEDOK) 1232 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE); 1233 if (irq_status[0] & IMR_BKDOK) 1234 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK); 1235 if (irq_status[0] & IMR_VODOK) 1236 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO); 1237 if (irq_status[0] & IMR_VIDOK) 1238 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI); 1239 if (irq_status[3] & IMR_H2CDOK) 1240 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C); 1241 if (irq_status[0] & IMR_ROK) { 1242 rtw_pci_rx_isr(rtwdev); 1243 rx = true; 1244 } 1245 if (unlikely(irq_status[0] & IMR_C2HCMD)) 1246 rtw_fw_c2h_cmd_isr(rtwdev); 1247 1248 /* all of the jobs for this interrupt have been done */ 1249 if (rtwpci->running) 1250 rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); 1251 spin_unlock_bh(&rtwpci->irq_lock); 1252 1253 return IRQ_HANDLED; 1254 } 1255 1256 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev, 1257 struct pci_dev *pdev) 1258 { 1259 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1260 unsigned long len; 1261 u8 bar_id = 2; 1262 int ret; 1263 1264 ret = pci_request_regions(pdev, KBUILD_MODNAME); 1265 if (ret) { 1266 rtw_err(rtwdev, "failed to request pci regions\n"); 1267 return ret; 1268 } 1269 1270 #if defined(__FreeBSD__) 1271 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 1272 if (ret) { 1273 rtw_err(rtwdev, "failed to set dma mask to 32-bit\n"); 1274 goto err_release_regions; 1275 } 1276 1277 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); 1278 if (ret) { 1279 rtw_err(rtwdev, "failed to set consistent dma mask to 32-bit\n"); 1280 goto err_release_regions; 1281 } 1282 #endif 1283 1284 len = pci_resource_len(pdev, bar_id); 1285 #if defined(__FreeBSD__) 1286 linuxkpi_pcim_want_to_use_bus_functions(pdev); 1287 #endif 1288 rtwpci->mmap = pci_iomap(pdev, bar_id, len); 1289 if (!rtwpci->mmap) { 1290 pci_release_regions(pdev); 1291 rtw_err(rtwdev, "failed to map pci memory\n"); 1292 return -ENOMEM; 1293 } 1294 1295 return 0; 1296 #if defined(__FreeBSD__) 1297 err_release_regions: 1298 pci_release_regions(pdev); 1299 return ret; 1300 #endif 1301 } 1302 1303 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev, 1304 struct pci_dev *pdev) 1305 { 1306 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1307 1308 if (rtwpci->mmap) { 1309 pci_iounmap(pdev, rtwpci->mmap); 1310 pci_release_regions(pdev); 1311 } 1312 } 1313 1314 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data) 1315 { 1316 u16 write_addr; 1317 u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK); 1318 u8 flag; 1319 u8 cnt; 1320 1321 write_addr = addr & BITS_DBI_ADDR_MASK; 1322 write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN); 1323 rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data); 1324 rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr); 1325 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16); 1326 1327 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1328 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1329 if (flag == 0) 1330 return; 1331 1332 udelay(10); 1333 } 1334 1335 WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr); 1336 } 1337 1338 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value) 1339 { 1340 u16 read_addr = addr & BITS_DBI_ADDR_MASK; 1341 u8 flag; 1342 u8 cnt; 1343 1344 rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr); 1345 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16); 1346 1347 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1348 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1349 if (flag == 0) { 1350 read_addr = REG_DBI_RDATA_V1 + (addr & 3); 1351 *value = rtw_read8(rtwdev, read_addr); 1352 return 0; 1353 } 1354 1355 udelay(10); 1356 } 1357 1358 WARN(1, "failed to read DBI register, addr=0x%04x\n", addr); 1359 return -EIO; 1360 } 1361 1362 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1) 1363 { 1364 u8 page; 1365 u8 wflag; 1366 u8 cnt; 1367 1368 rtw_write16(rtwdev, REG_MDIO_V1, data); 1369 1370 page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1; 1371 page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2; 1372 rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK); 1373 rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page); 1374 rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1); 1375 1376 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1377 wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG, 1378 BIT_MDIO_WFLAG_V1); 1379 if (wflag == 0) 1380 return; 1381 1382 udelay(10); 1383 } 1384 1385 WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr); 1386 } 1387 1388 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable) 1389 { 1390 u8 value; 1391 int ret; 1392 1393 if (rtw_pci_disable_aspm) 1394 return; 1395 1396 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1397 if (ret) { 1398 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1399 return; 1400 } 1401 1402 if (enable) 1403 value |= BIT_CLKREQ_SW_EN; 1404 else 1405 value &= ~BIT_CLKREQ_SW_EN; 1406 1407 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1408 } 1409 1410 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable) 1411 { 1412 u8 value; 1413 int ret; 1414 1415 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1416 if (ret) { 1417 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1418 return; 1419 } 1420 1421 if (enable) 1422 value &= ~BIT_CLKREQ_N_PAD; 1423 else 1424 value |= BIT_CLKREQ_N_PAD; 1425 1426 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1427 } 1428 1429 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable) 1430 { 1431 u8 value; 1432 int ret; 1433 1434 if (rtw_pci_disable_aspm) 1435 return; 1436 1437 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1438 if (ret) { 1439 rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret); 1440 return; 1441 } 1442 1443 if (enable) 1444 value |= BIT_L1_SW_EN; 1445 else 1446 value &= ~BIT_L1_SW_EN; 1447 1448 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1449 } 1450 1451 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter) 1452 { 1453 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1454 1455 /* Like CLKREQ, ASPM is also implemented by two HW modules, and can 1456 * only be enabled when host supports it. 1457 * 1458 * And ASPM mechanism should be enabled when driver/firmware enters 1459 * power save mode, without having heavy traffic. Because we've 1460 * experienced some inter-operability issues that the link tends 1461 * to enter L1 state on the fly even when driver is having high 1462 * throughput. This is probably because the ASPM behavior slightly 1463 * varies from different SOC. 1464 */ 1465 if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)) 1466 return; 1467 1468 if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) || 1469 (!enter && atomic_inc_return(&rtwpci->link_usage) == 1)) 1470 rtw_pci_aspm_set(rtwdev, enter); 1471 } 1472 1473 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) 1474 { 1475 const struct rtw_chip_info *chip = rtwdev->chip; 1476 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1477 struct pci_dev *pdev = rtwpci->pdev; 1478 u16 link_ctrl; 1479 int ret; 1480 1481 /* RTL8822CE has enabled REFCLK auto calibration, it does not need 1482 * to add clock delay to cover the REFCLK timing gap. 1483 */ 1484 if (chip->id == RTW_CHIP_TYPE_8822C) 1485 rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0); 1486 1487 /* Though there is standard PCIE configuration space to set the 1488 * link control register, but by Realtek's design, driver should 1489 * check if host supports CLKREQ/ASPM to enable the HW module. 1490 * 1491 * These functions are implemented by two HW modules associated, 1492 * one is responsible to access PCIE configuration space to 1493 * follow the host settings, and another is in charge of doing 1494 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes 1495 * the host does not support it, and due to some reasons or wrong 1496 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device 1497 * loss if HW misbehaves on the link. 1498 * 1499 * Hence it's designed that driver should first check the PCIE 1500 * configuration space is sync'ed and enabled, then driver can turn 1501 * on the other module that is actually working on the mechanism. 1502 */ 1503 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl); 1504 if (ret) { 1505 rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret); 1506 return; 1507 } 1508 1509 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN) 1510 rtw_pci_clkreq_set(rtwdev, true); 1511 1512 rtwpci->link_ctrl = link_ctrl; 1513 } 1514 1515 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) 1516 { 1517 const struct rtw_chip_info *chip = rtwdev->chip; 1518 1519 switch (chip->id) { 1520 case RTW_CHIP_TYPE_8822C: 1521 if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D) 1522 rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG, 1523 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1); 1524 break; 1525 default: 1526 break; 1527 } 1528 } 1529 1530 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev) 1531 { 1532 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1533 const struct rtw_chip_info *chip = rtwdev->chip; 1534 struct rtw_efuse *efuse = &rtwdev->efuse; 1535 struct pci_dev *pdev = rtwpci->pdev; 1536 const struct rtw_intf_phy_para *para; 1537 u16 cut; 1538 u16 value; 1539 u16 offset; 1540 int i; 1541 int ret; 1542 1543 cut = BIT(0) << rtwdev->hal.cut_version; 1544 1545 for (i = 0; i < chip->intf_table->n_gen1_para; i++) { 1546 para = &chip->intf_table->gen1_para[i]; 1547 if (!(para->cut_mask & cut)) 1548 continue; 1549 if (para->offset == 0xffff) 1550 break; 1551 offset = para->offset; 1552 value = para->value; 1553 if (para->ip_sel == RTW_IP_SEL_PHY) 1554 rtw_mdio_write(rtwdev, offset, value, true); 1555 else 1556 rtw_dbi_write8(rtwdev, offset, value); 1557 } 1558 1559 for (i = 0; i < chip->intf_table->n_gen2_para; i++) { 1560 para = &chip->intf_table->gen2_para[i]; 1561 if (!(para->cut_mask & cut)) 1562 continue; 1563 if (para->offset == 0xffff) 1564 break; 1565 offset = para->offset; 1566 value = para->value; 1567 if (para->ip_sel == RTW_IP_SEL_PHY) 1568 rtw_mdio_write(rtwdev, offset, value, false); 1569 else 1570 rtw_dbi_write8(rtwdev, offset, value); 1571 } 1572 1573 rtw_pci_link_cfg(rtwdev); 1574 1575 /* Disable 8821ce completion timeout by default */ 1576 if (chip->id == RTW_CHIP_TYPE_8821C) { 1577 ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, 1578 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); 1579 if (ret) 1580 rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n", 1581 ret); 1582 } 1583 1584 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 5) 1585 rtw_write32_mask(rtwdev, REG_ANAPARSW_MAC_0, BIT_CF_L_V2, 0x1); 1586 } 1587 1588 static int __maybe_unused rtw_pci_suspend(struct device *dev) 1589 { 1590 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1591 struct rtw_dev *rtwdev = hw->priv; 1592 const struct rtw_chip_info *chip = rtwdev->chip; 1593 struct rtw_efuse *efuse = &rtwdev->efuse; 1594 1595 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1596 rtw_pci_clkreq_pad_low(rtwdev, true); 1597 return 0; 1598 } 1599 1600 static int __maybe_unused rtw_pci_resume(struct device *dev) 1601 { 1602 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1603 struct rtw_dev *rtwdev = hw->priv; 1604 const struct rtw_chip_info *chip = rtwdev->chip; 1605 struct rtw_efuse *efuse = &rtwdev->efuse; 1606 1607 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1608 rtw_pci_clkreq_pad_low(rtwdev, false); 1609 return 0; 1610 } 1611 1612 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume); 1613 EXPORT_SYMBOL(rtw_pm_ops); 1614 1615 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1616 { 1617 int ret; 1618 1619 ret = pci_enable_device(pdev); 1620 if (ret) { 1621 rtw_err(rtwdev, "failed to enable pci device\n"); 1622 return ret; 1623 } 1624 1625 pci_set_master(pdev); 1626 pci_set_drvdata(pdev, rtwdev->hw); 1627 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); 1628 1629 return 0; 1630 } 1631 1632 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1633 { 1634 pci_disable_device(pdev); 1635 } 1636 1637 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1638 { 1639 struct rtw_pci *rtwpci; 1640 int ret; 1641 1642 rtwpci = (struct rtw_pci *)rtwdev->priv; 1643 rtwpci->pdev = pdev; 1644 1645 /* after this driver can access to hw registers */ 1646 ret = rtw_pci_io_mapping(rtwdev, pdev); 1647 if (ret) { 1648 rtw_err(rtwdev, "failed to request pci io region\n"); 1649 goto err_out; 1650 } 1651 1652 ret = rtw_pci_init(rtwdev); 1653 if (ret) { 1654 rtw_err(rtwdev, "failed to allocate pci resources\n"); 1655 goto err_io_unmap; 1656 } 1657 1658 return 0; 1659 1660 err_io_unmap: 1661 rtw_pci_io_unmapping(rtwdev, pdev); 1662 1663 err_out: 1664 return ret; 1665 } 1666 1667 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1668 { 1669 rtw_pci_deinit(rtwdev); 1670 rtw_pci_io_unmapping(rtwdev, pdev); 1671 } 1672 1673 static struct rtw_hci_ops rtw_pci_ops = { 1674 .tx_write = rtw_pci_tx_write, 1675 .tx_kick_off = rtw_pci_tx_kick_off, 1676 .flush_queues = rtw_pci_flush_queues, 1677 .setup = rtw_pci_setup, 1678 .start = rtw_pci_start, 1679 .stop = rtw_pci_stop, 1680 .deep_ps = rtw_pci_deep_ps, 1681 .link_ps = rtw_pci_link_ps, 1682 .interface_cfg = rtw_pci_interface_cfg, 1683 .dynamic_rx_agg = NULL, 1684 1685 .read8 = rtw_pci_read8, 1686 .read16 = rtw_pci_read16, 1687 .read32 = rtw_pci_read32, 1688 .write8 = rtw_pci_write8, 1689 .write16 = rtw_pci_write16, 1690 .write32 = rtw_pci_write32, 1691 .write_data_rsvd_page = rtw_pci_write_data_rsvd_page, 1692 .write_data_h2c = rtw_pci_write_data_h2c, 1693 }; 1694 1695 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1696 { 1697 unsigned int flags = PCI_IRQ_INTX; 1698 int ret; 1699 1700 if (!rtw_disable_msi) 1701 flags |= PCI_IRQ_MSI; 1702 1703 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); 1704 if (ret < 0) { 1705 rtw_err(rtwdev, "failed to alloc PCI irq vectors\n"); 1706 return ret; 1707 } 1708 1709 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, 1710 rtw_pci_interrupt_handler, 1711 rtw_pci_interrupt_threadfn, 1712 IRQF_SHARED, KBUILD_MODNAME, rtwdev); 1713 if (ret) { 1714 rtw_err(rtwdev, "failed to request irq %d\n", ret); 1715 pci_free_irq_vectors(pdev); 1716 } 1717 1718 return ret; 1719 } 1720 1721 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1722 { 1723 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); 1724 pci_free_irq_vectors(pdev); 1725 } 1726 1727 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget) 1728 { 1729 struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi); 1730 struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev, 1731 priv); 1732 int work_done = 0; 1733 1734 if (rtwpci->rx_no_aspm) 1735 rtw_pci_link_ps(rtwdev, false); 1736 1737 while (work_done < budget) { 1738 u32 work_done_once; 1739 1740 work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU, 1741 budget - work_done); 1742 if (work_done_once == 0) 1743 break; 1744 work_done += work_done_once; 1745 } 1746 if (work_done < budget) { 1747 napi_complete_done(napi, work_done); 1748 spin_lock_bh(&rtwpci->irq_lock); 1749 if (rtwpci->running) 1750 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 1751 spin_unlock_bh(&rtwpci->irq_lock); 1752 /* When ISR happens during polling and before napi_complete 1753 * while no further data is received. Data on the dma_ring will 1754 * not be processed immediately. Check whether dma ring is 1755 * empty and perform napi_schedule accordingly. 1756 */ 1757 if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci)) 1758 napi_schedule(napi); 1759 } 1760 if (rtwpci->rx_no_aspm) 1761 rtw_pci_link_ps(rtwdev, true); 1762 1763 return work_done; 1764 } 1765 1766 static int rtw_pci_napi_init(struct rtw_dev *rtwdev) 1767 { 1768 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1769 1770 rtwpci->netdev = alloc_netdev_dummy(0); 1771 if (!rtwpci->netdev) 1772 return -ENOMEM; 1773 1774 netif_napi_add(rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll); 1775 return 0; 1776 } 1777 1778 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) 1779 { 1780 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1781 1782 rtw_pci_napi_stop(rtwdev); 1783 netif_napi_del(&rtwpci->napi); 1784 free_netdev(rtwpci->netdev); 1785 } 1786 1787 int rtw_pci_probe(struct pci_dev *pdev, 1788 const struct pci_device_id *id) 1789 { 1790 struct pci_dev *bridge = pci_upstream_bridge(pdev); 1791 struct ieee80211_hw *hw; 1792 struct rtw_dev *rtwdev; 1793 struct rtw_pci *rtwpci; 1794 int drv_data_size; 1795 int ret; 1796 1797 drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci); 1798 hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); 1799 if (!hw) { 1800 dev_err(&pdev->dev, "failed to allocate hw\n"); 1801 return -ENOMEM; 1802 } 1803 1804 rtwdev = hw->priv; 1805 rtwdev->hw = hw; 1806 rtwdev->dev = &pdev->dev; 1807 rtwdev->chip = (struct rtw_chip_info *)id->driver_data; 1808 rtwdev->hci.ops = &rtw_pci_ops; 1809 rtwdev->hci.type = RTW_HCI_TYPE_PCIE; 1810 1811 rtwpci = (struct rtw_pci *)rtwdev->priv; 1812 atomic_set(&rtwpci->link_usage, 1); 1813 1814 ret = rtw_core_init(rtwdev); 1815 if (ret) 1816 goto err_release_hw; 1817 1818 rtw_dbg(rtwdev, RTW_DBG_PCI, 1819 "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n", 1820 pdev->vendor, pdev->device, pdev->revision); 1821 1822 ret = rtw_pci_claim(rtwdev, pdev); 1823 if (ret) { 1824 rtw_err(rtwdev, "failed to claim pci device\n"); 1825 goto err_deinit_core; 1826 } 1827 1828 ret = rtw_pci_setup_resource(rtwdev, pdev); 1829 if (ret) { 1830 rtw_err(rtwdev, "failed to setup pci resources\n"); 1831 goto err_pci_declaim; 1832 } 1833 1834 ret = rtw_pci_napi_init(rtwdev); 1835 if (ret) { 1836 rtw_err(rtwdev, "failed to setup NAPI\n"); 1837 goto err_pci_declaim; 1838 } 1839 1840 ret = rtw_chip_info_setup(rtwdev); 1841 if (ret) { 1842 rtw_err(rtwdev, "failed to setup chip information\n"); 1843 goto err_destroy_pci; 1844 } 1845 1846 /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */ 1847 if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL) 1848 rtwpci->rx_no_aspm = true; 1849 1850 rtw_pci_phy_cfg(rtwdev); 1851 1852 ret = rtw_register_hw(rtwdev, hw); 1853 if (ret) { 1854 rtw_err(rtwdev, "failed to register hw\n"); 1855 goto err_destroy_pci; 1856 } 1857 1858 ret = rtw_pci_request_irq(rtwdev, pdev); 1859 if (ret) { 1860 ieee80211_unregister_hw(hw); 1861 goto err_destroy_pci; 1862 } 1863 1864 return 0; 1865 1866 err_destroy_pci: 1867 rtw_pci_napi_deinit(rtwdev); 1868 rtw_pci_destroy(rtwdev, pdev); 1869 1870 err_pci_declaim: 1871 rtw_pci_declaim(rtwdev, pdev); 1872 1873 err_deinit_core: 1874 rtw_core_deinit(rtwdev); 1875 1876 err_release_hw: 1877 ieee80211_free_hw(hw); 1878 1879 return ret; 1880 } 1881 EXPORT_SYMBOL(rtw_pci_probe); 1882 1883 void rtw_pci_remove(struct pci_dev *pdev) 1884 { 1885 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1886 struct rtw_dev *rtwdev; 1887 struct rtw_pci *rtwpci; 1888 1889 if (!hw) 1890 return; 1891 1892 rtwdev = hw->priv; 1893 rtwpci = (struct rtw_pci *)rtwdev->priv; 1894 1895 rtw_unregister_hw(rtwdev, hw); 1896 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1897 rtw_pci_napi_deinit(rtwdev); 1898 rtw_pci_destroy(rtwdev, pdev); 1899 rtw_pci_declaim(rtwdev, pdev); 1900 rtw_pci_free_irq(rtwdev, pdev); 1901 rtw_core_deinit(rtwdev); 1902 ieee80211_free_hw(hw); 1903 } 1904 EXPORT_SYMBOL(rtw_pci_remove); 1905 1906 void rtw_pci_shutdown(struct pci_dev *pdev) 1907 { 1908 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1909 struct rtw_dev *rtwdev; 1910 const struct rtw_chip_info *chip; 1911 1912 if (!hw) 1913 return; 1914 1915 rtwdev = hw->priv; 1916 chip = rtwdev->chip; 1917 1918 if (chip->ops->shutdown) 1919 chip->ops->shutdown(rtwdev); 1920 1921 pci_set_power_state(pdev, PCI_D3hot); 1922 } 1923 EXPORT_SYMBOL(rtw_pci_shutdown); 1924 1925 MODULE_AUTHOR("Realtek Corporation"); 1926 MODULE_DESCRIPTION("Realtek PCI 802.11ac wireless driver"); 1927 MODULE_LICENSE("Dual BSD/GPL"); 1928 #if defined(__FreeBSD__) 1929 MODULE_VERSION(rtw_pci, 1); 1930 MODULE_DEPEND(rtw_pci, linuxkpi, 1, 1, 1); 1931 MODULE_DEPEND(rtw_pci, linuxkpi_wlan, 1, 1, 1); 1932 #ifdef CONFIG_RTW88_DEBUGFS 1933 MODULE_DEPEND(rtw_pci, lindebugfs, 1, 1, 1); 1934 #endif 1935 #endif 1936