1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright(c) 2018-2019 Realtek Corporation 3 */ 4 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include "main.h" 8 #include "pci.h" 9 #include "reg.h" 10 #include "tx.h" 11 #include "rx.h" 12 #include "fw.h" 13 #include "ps.h" 14 #include "debug.h" 15 16 static bool rtw_disable_msi; 17 static bool rtw_pci_disable_aspm; 18 module_param_named(disable_msi, rtw_disable_msi, bool, 0644); 19 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644); 20 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support"); 21 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support"); 22 23 static u32 rtw_pci_tx_queue_idx_addr[] = { 24 [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ, 25 [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ, 26 [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ, 27 [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ, 28 [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ, 29 [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q, 30 [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ, 31 }; 32 33 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, 34 enum rtw_tx_queue_type queue) 35 { 36 switch (queue) { 37 case RTW_TX_QUEUE_BCN: 38 return TX_DESC_QSEL_BEACON; 39 case RTW_TX_QUEUE_H2C: 40 return TX_DESC_QSEL_H2C; 41 case RTW_TX_QUEUE_MGMT: 42 return TX_DESC_QSEL_MGMT; 43 case RTW_TX_QUEUE_HI0: 44 return TX_DESC_QSEL_HIGH; 45 default: 46 return skb->priority; 47 } 48 }; 49 50 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr) 51 { 52 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 53 54 return readb(rtwpci->mmap + addr); 55 } 56 57 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr) 58 { 59 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 60 61 return readw(rtwpci->mmap + addr); 62 } 63 64 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr) 65 { 66 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 67 68 return readl(rtwpci->mmap + addr); 69 } 70 71 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val) 72 { 73 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 74 75 writeb(val, rtwpci->mmap + addr); 76 } 77 78 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val) 79 { 80 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 81 82 writew(val, rtwpci->mmap + addr); 83 } 84 85 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val) 86 { 87 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 88 89 writel(val, rtwpci->mmap + addr); 90 } 91 92 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev, 93 struct rtw_pci_tx_ring *tx_ring) 94 { 95 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 96 struct rtw_pci_tx_data *tx_data; 97 struct sk_buff *skb, *tmp; 98 dma_addr_t dma; 99 100 /* free every skb remained in tx list */ 101 skb_queue_walk_safe(&tx_ring->queue, skb, tmp) { 102 __skb_unlink(skb, &tx_ring->queue); 103 tx_data = rtw_pci_get_tx_data(skb); 104 dma = tx_data->dma; 105 106 dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE); 107 dev_kfree_skb_any(skb); 108 } 109 } 110 111 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev, 112 struct rtw_pci_tx_ring *tx_ring) 113 { 114 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 115 u8 *head = tx_ring->r.head; 116 u32 len = tx_ring->r.len; 117 int ring_sz = len * tx_ring->r.desc_size; 118 119 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 120 121 /* free the ring itself */ 122 dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma); 123 tx_ring->r.head = NULL; 124 } 125 126 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev, 127 struct rtw_pci_rx_ring *rx_ring) 128 { 129 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 130 struct sk_buff *skb; 131 int buf_sz = RTK_PCI_RX_BUF_SIZE; 132 dma_addr_t dma; 133 int i; 134 135 for (i = 0; i < rx_ring->r.len; i++) { 136 skb = rx_ring->buf[i]; 137 if (!skb) 138 continue; 139 140 dma = *((dma_addr_t *)skb->cb); 141 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 142 dev_kfree_skb(skb); 143 rx_ring->buf[i] = NULL; 144 } 145 } 146 147 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev, 148 struct rtw_pci_rx_ring *rx_ring) 149 { 150 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 151 u8 *head = rx_ring->r.head; 152 int ring_sz = rx_ring->r.desc_size * rx_ring->r.len; 153 154 rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring); 155 156 dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma); 157 } 158 159 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev) 160 { 161 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 162 struct rtw_pci_tx_ring *tx_ring; 163 struct rtw_pci_rx_ring *rx_ring; 164 int i; 165 166 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 167 tx_ring = &rtwpci->tx_rings[i]; 168 rtw_pci_free_tx_ring(rtwdev, tx_ring); 169 } 170 171 for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) { 172 rx_ring = &rtwpci->rx_rings[i]; 173 rtw_pci_free_rx_ring(rtwdev, rx_ring); 174 } 175 } 176 177 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev, 178 struct rtw_pci_tx_ring *tx_ring, 179 u8 desc_size, u32 len) 180 { 181 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 182 int ring_sz = desc_size * len; 183 dma_addr_t dma; 184 u8 *head; 185 186 if (len > TRX_BD_IDX_MASK) { 187 rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len); 188 return -EINVAL; 189 } 190 191 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 192 if (!head) { 193 rtw_err(rtwdev, "failed to allocate tx ring\n"); 194 return -ENOMEM; 195 } 196 197 skb_queue_head_init(&tx_ring->queue); 198 tx_ring->r.head = head; 199 tx_ring->r.dma = dma; 200 tx_ring->r.len = len; 201 tx_ring->r.desc_size = desc_size; 202 tx_ring->r.wp = 0; 203 tx_ring->r.rp = 0; 204 205 return 0; 206 } 207 208 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb, 209 struct rtw_pci_rx_ring *rx_ring, 210 u32 idx, u32 desc_sz) 211 { 212 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 213 struct rtw_pci_rx_buffer_desc *buf_desc; 214 int buf_sz = RTK_PCI_RX_BUF_SIZE; 215 dma_addr_t dma; 216 217 if (!skb) 218 return -EINVAL; 219 220 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); 221 if (dma_mapping_error(&pdev->dev, dma)) 222 return -EBUSY; 223 224 *((dma_addr_t *)skb->cb) = dma; 225 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 226 idx * desc_sz); 227 memset(buf_desc, 0, sizeof(*buf_desc)); 228 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 229 buf_desc->dma = cpu_to_le32(dma); 230 231 return 0; 232 } 233 234 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma, 235 struct rtw_pci_rx_ring *rx_ring, 236 u32 idx, u32 desc_sz) 237 { 238 struct device *dev = rtwdev->dev; 239 struct rtw_pci_rx_buffer_desc *buf_desc; 240 int buf_sz = RTK_PCI_RX_BUF_SIZE; 241 242 dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE); 243 244 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 245 idx * desc_sz); 246 memset(buf_desc, 0, sizeof(*buf_desc)); 247 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 248 buf_desc->dma = cpu_to_le32(dma); 249 } 250 251 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev, 252 struct rtw_pci_rx_ring *rx_ring, 253 u8 desc_size, u32 len) 254 { 255 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 256 struct sk_buff *skb = NULL; 257 dma_addr_t dma; 258 u8 *head; 259 int ring_sz = desc_size * len; 260 int buf_sz = RTK_PCI_RX_BUF_SIZE; 261 int i, allocated; 262 int ret = 0; 263 264 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 265 if (!head) { 266 rtw_err(rtwdev, "failed to allocate rx ring\n"); 267 return -ENOMEM; 268 } 269 rx_ring->r.head = head; 270 271 for (i = 0; i < len; i++) { 272 skb = dev_alloc_skb(buf_sz); 273 if (!skb) { 274 allocated = i; 275 ret = -ENOMEM; 276 goto err_out; 277 } 278 279 memset(skb->data, 0, buf_sz); 280 rx_ring->buf[i] = skb; 281 ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size); 282 if (ret) { 283 allocated = i; 284 dev_kfree_skb_any(skb); 285 goto err_out; 286 } 287 } 288 289 rx_ring->r.dma = dma; 290 rx_ring->r.len = len; 291 rx_ring->r.desc_size = desc_size; 292 rx_ring->r.wp = 0; 293 rx_ring->r.rp = 0; 294 295 return 0; 296 297 err_out: 298 for (i = 0; i < allocated; i++) { 299 skb = rx_ring->buf[i]; 300 if (!skb) 301 continue; 302 dma = *((dma_addr_t *)skb->cb); 303 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 304 dev_kfree_skb_any(skb); 305 rx_ring->buf[i] = NULL; 306 } 307 dma_free_coherent(&pdev->dev, ring_sz, head, dma); 308 309 rtw_err(rtwdev, "failed to init rx buffer\n"); 310 311 return ret; 312 } 313 314 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev) 315 { 316 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 317 struct rtw_pci_tx_ring *tx_ring; 318 struct rtw_pci_rx_ring *rx_ring; 319 const struct rtw_chip_info *chip = rtwdev->chip; 320 int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0; 321 int tx_desc_size, rx_desc_size; 322 u32 len; 323 int ret; 324 325 tx_desc_size = chip->tx_buf_desc_sz; 326 327 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 328 tx_ring = &rtwpci->tx_rings[i]; 329 len = max_num_of_tx_queue(i); 330 ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len); 331 if (ret) 332 goto out; 333 } 334 335 rx_desc_size = chip->rx_buf_desc_sz; 336 337 for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) { 338 rx_ring = &rtwpci->rx_rings[j]; 339 ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size, 340 RTK_MAX_RX_DESC_NUM); 341 if (ret) 342 goto out; 343 } 344 345 return 0; 346 347 out: 348 tx_alloced = i; 349 for (i = 0; i < tx_alloced; i++) { 350 tx_ring = &rtwpci->tx_rings[i]; 351 rtw_pci_free_tx_ring(rtwdev, tx_ring); 352 } 353 354 rx_alloced = j; 355 for (j = 0; j < rx_alloced; j++) { 356 rx_ring = &rtwpci->rx_rings[j]; 357 rtw_pci_free_rx_ring(rtwdev, rx_ring); 358 } 359 360 return ret; 361 } 362 363 static void rtw_pci_deinit(struct rtw_dev *rtwdev) 364 { 365 rtw_pci_free_trx_ring(rtwdev); 366 } 367 368 static int rtw_pci_init(struct rtw_dev *rtwdev) 369 { 370 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 371 int ret = 0; 372 373 rtwpci->irq_mask[0] = IMR_HIGHDOK | 374 IMR_MGNTDOK | 375 IMR_BKDOK | 376 IMR_BEDOK | 377 IMR_VIDOK | 378 IMR_VODOK | 379 IMR_ROK | 380 IMR_BCNDMAINT_E | 381 IMR_C2HCMD | 382 0; 383 rtwpci->irq_mask[1] = IMR_TXFOVW | 384 0; 385 rtwpci->irq_mask[3] = IMR_H2CDOK | 386 0; 387 spin_lock_init(&rtwpci->irq_lock); 388 spin_lock_init(&rtwpci->hwirq_lock); 389 ret = rtw_pci_init_trx_ring(rtwdev); 390 391 return ret; 392 } 393 394 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev) 395 { 396 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 397 u32 len; 398 u8 tmp; 399 dma_addr_t dma; 400 401 tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3); 402 rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7); 403 404 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma; 405 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma); 406 407 if (!rtw_chip_wcpu_11n(rtwdev)) { 408 len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len; 409 dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma; 410 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0; 411 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0; 412 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK); 413 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma); 414 } 415 416 len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len; 417 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma; 418 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0; 419 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0; 420 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK); 421 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma); 422 423 len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len; 424 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma; 425 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0; 426 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0; 427 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK); 428 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma); 429 430 len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len; 431 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma; 432 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0; 433 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0; 434 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK); 435 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma); 436 437 len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len; 438 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma; 439 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0; 440 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0; 441 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK); 442 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma); 443 444 len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len; 445 dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma; 446 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0; 447 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0; 448 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK); 449 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma); 450 451 len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len; 452 dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma; 453 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0; 454 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0; 455 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK); 456 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma); 457 458 len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len; 459 dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma; 460 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0; 461 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0; 462 rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK); 463 rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma); 464 465 /* reset read/write point */ 466 rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff); 467 468 /* reset H2C Queue index in a single write */ 469 if (rtw_chip_wcpu_11ac(rtwdev)) 470 rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR, 471 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX); 472 } 473 474 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev) 475 { 476 rtw_pci_reset_buf_desc(rtwdev); 477 } 478 479 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev, 480 struct rtw_pci *rtwpci, bool exclude_rx) 481 { 482 unsigned long flags; 483 u32 imr0_unmask = exclude_rx ? IMR_ROK : 0; 484 485 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 486 487 rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask); 488 rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]); 489 if (rtw_chip_wcpu_11ac(rtwdev)) 490 rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]); 491 492 rtwpci->irq_enabled = true; 493 494 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 495 } 496 497 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev, 498 struct rtw_pci *rtwpci) 499 { 500 unsigned long flags; 501 502 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 503 504 if (!rtwpci->irq_enabled) 505 goto out; 506 507 rtw_write32(rtwdev, RTK_PCI_HIMR0, 0); 508 rtw_write32(rtwdev, RTK_PCI_HIMR1, 0); 509 if (rtw_chip_wcpu_11ac(rtwdev)) 510 rtw_write32(rtwdev, RTK_PCI_HIMR3, 0); 511 512 rtwpci->irq_enabled = false; 513 514 out: 515 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 516 } 517 518 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 519 { 520 /* reset dma and rx tag */ 521 rtw_write32_set(rtwdev, RTK_PCI_CTRL, 522 BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN); 523 rtwpci->rx_tag = 0; 524 } 525 526 static int rtw_pci_setup(struct rtw_dev *rtwdev) 527 { 528 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 529 530 rtw_pci_reset_trx_ring(rtwdev); 531 rtw_pci_dma_reset(rtwdev, rtwpci); 532 533 return 0; 534 } 535 536 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 537 { 538 struct rtw_pci_tx_ring *tx_ring; 539 enum rtw_tx_queue_type queue; 540 541 rtw_pci_reset_trx_ring(rtwdev); 542 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 543 tx_ring = &rtwpci->tx_rings[queue]; 544 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 545 } 546 } 547 548 static void rtw_pci_napi_start(struct rtw_dev *rtwdev) 549 { 550 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 551 552 if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 553 return; 554 555 napi_enable(&rtwpci->napi); 556 } 557 558 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev) 559 { 560 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 561 562 if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 563 return; 564 565 napi_synchronize(&rtwpci->napi); 566 napi_disable(&rtwpci->napi); 567 } 568 569 static int rtw_pci_start(struct rtw_dev *rtwdev) 570 { 571 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 572 573 rtw_pci_napi_start(rtwdev); 574 575 spin_lock_bh(&rtwpci->irq_lock); 576 rtwpci->running = true; 577 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 578 spin_unlock_bh(&rtwpci->irq_lock); 579 580 return 0; 581 } 582 583 static void rtw_pci_stop(struct rtw_dev *rtwdev) 584 { 585 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 586 struct pci_dev *pdev = rtwpci->pdev; 587 588 spin_lock_bh(&rtwpci->irq_lock); 589 rtwpci->running = false; 590 rtw_pci_disable_interrupt(rtwdev, rtwpci); 591 spin_unlock_bh(&rtwpci->irq_lock); 592 593 synchronize_irq(pdev->irq); 594 rtw_pci_napi_stop(rtwdev); 595 596 spin_lock_bh(&rtwpci->irq_lock); 597 rtw_pci_dma_release(rtwdev, rtwpci); 598 spin_unlock_bh(&rtwpci->irq_lock); 599 } 600 601 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev) 602 { 603 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 604 struct rtw_pci_tx_ring *tx_ring; 605 enum rtw_tx_queue_type queue; 606 bool tx_empty = true; 607 608 if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 609 goto enter_deep_ps; 610 611 lockdep_assert_held(&rtwpci->irq_lock); 612 613 /* Deep PS state is not allowed to TX-DMA */ 614 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 615 /* BCN queue is rsvd page, does not have DMA interrupt 616 * H2C queue is managed by firmware 617 */ 618 if (queue == RTW_TX_QUEUE_BCN || 619 queue == RTW_TX_QUEUE_H2C) 620 continue; 621 622 tx_ring = &rtwpci->tx_rings[queue]; 623 624 /* check if there is any skb DMAing */ 625 if (skb_queue_len(&tx_ring->queue)) { 626 tx_empty = false; 627 break; 628 } 629 } 630 631 if (!tx_empty) { 632 rtw_dbg(rtwdev, RTW_DBG_PS, 633 "TX path not empty, cannot enter deep power save state\n"); 634 return; 635 } 636 enter_deep_ps: 637 set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags); 638 rtw_power_mode_change(rtwdev, true); 639 } 640 641 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev) 642 { 643 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 644 645 lockdep_assert_held(&rtwpci->irq_lock); 646 647 if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 648 rtw_power_mode_change(rtwdev, false); 649 } 650 651 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter) 652 { 653 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 654 655 spin_lock_bh(&rtwpci->irq_lock); 656 657 if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 658 rtw_pci_deep_ps_enter(rtwdev); 659 660 if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 661 rtw_pci_deep_ps_leave(rtwdev); 662 663 spin_unlock_bh(&rtwpci->irq_lock); 664 } 665 666 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci, 667 struct rtw_pci_tx_ring *ring) 668 { 669 struct sk_buff *prev = skb_dequeue(&ring->queue); 670 struct rtw_pci_tx_data *tx_data; 671 dma_addr_t dma; 672 673 if (!prev) 674 return; 675 676 tx_data = rtw_pci_get_tx_data(prev); 677 dma = tx_data->dma; 678 dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE); 679 dev_kfree_skb_any(prev); 680 } 681 682 static void rtw_pci_dma_check(struct rtw_dev *rtwdev, 683 struct rtw_pci_rx_ring *rx_ring, 684 u32 idx) 685 { 686 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 687 const struct rtw_chip_info *chip = rtwdev->chip; 688 struct rtw_pci_rx_buffer_desc *buf_desc; 689 u32 desc_sz = chip->rx_buf_desc_sz; 690 u16 total_pkt_size; 691 692 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 693 idx * desc_sz); 694 total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size); 695 696 /* rx tag mismatch, throw a warning */ 697 if (total_pkt_size != rtwpci->rx_tag) 698 rtw_warn(rtwdev, "pci bus timeout, check dma status\n"); 699 700 rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX; 701 } 702 703 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q) 704 { 705 u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q]; 706 u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2); 707 708 return FIELD_GET(TRX_BD_IDX_MASK, bd_idx); 709 } 710 711 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop) 712 { 713 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 714 struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q]; 715 u32 cur_rp; 716 u8 i; 717 718 /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a 719 * bit dynamic, it's hard to define a reasonable fixed total timeout to 720 * use read_poll_timeout* helper. Instead, we can ensure a reasonable 721 * polling times, so we just use for loop with udelay here. 722 */ 723 for (i = 0; i < 30; i++) { 724 cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q); 725 if (cur_rp == ring->r.wp) 726 return; 727 728 udelay(1); 729 } 730 731 if (!drop) 732 rtw_dbg(rtwdev, RTW_DBG_UNEXP, 733 "timed out to flush pci tx ring[%d]\n", pci_q); 734 } 735 736 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues, 737 bool drop) 738 { 739 u8 q; 740 741 for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) { 742 /* Unnecessary to flush BCN, H2C and HI tx queues. */ 743 if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C || 744 q == RTW_TX_QUEUE_HI0) 745 continue; 746 747 if (pci_queues & BIT(q)) 748 __pci_flush_queue(rtwdev, q, drop); 749 } 750 } 751 752 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop) 753 { 754 u32 pci_queues = 0; 755 u8 i; 756 757 /* If all of the hardware queues are requested to flush, 758 * flush all of the pci queues. 759 */ 760 if (queues == BIT(rtwdev->hw->queues) - 1) { 761 pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1; 762 } else { 763 for (i = 0; i < rtwdev->hw->queues; i++) 764 if (queues & BIT(i)) 765 pci_queues |= BIT(rtw_tx_ac_to_hwq(i)); 766 } 767 768 __rtw_pci_flush_queues(rtwdev, pci_queues, drop); 769 } 770 771 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, 772 enum rtw_tx_queue_type queue) 773 { 774 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 775 struct rtw_pci_tx_ring *ring; 776 u32 bd_idx; 777 778 ring = &rtwpci->tx_rings[queue]; 779 bd_idx = rtw_pci_tx_queue_idx_addr[queue]; 780 781 spin_lock_bh(&rtwpci->irq_lock); 782 if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 783 rtw_pci_deep_ps_leave(rtwdev); 784 rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK); 785 spin_unlock_bh(&rtwpci->irq_lock); 786 } 787 788 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev) 789 { 790 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 791 enum rtw_tx_queue_type queue; 792 793 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) 794 if (test_and_clear_bit(queue, rtwpci->tx_queued)) 795 rtw_pci_tx_kick_off_queue(rtwdev, queue); 796 } 797 798 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev, 799 struct rtw_tx_pkt_info *pkt_info, 800 struct sk_buff *skb, 801 enum rtw_tx_queue_type queue) 802 { 803 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 804 const struct rtw_chip_info *chip = rtwdev->chip; 805 struct rtw_pci_tx_ring *ring; 806 struct rtw_pci_tx_data *tx_data; 807 dma_addr_t dma; 808 u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz; 809 u32 tx_buf_desc_sz = chip->tx_buf_desc_sz; 810 u32 size; 811 u32 psb_len; 812 u8 *pkt_desc; 813 struct rtw_pci_tx_buffer_desc *buf_desc; 814 815 ring = &rtwpci->tx_rings[queue]; 816 817 size = skb->len; 818 819 if (queue == RTW_TX_QUEUE_BCN) 820 rtw_pci_release_rsvd_page(rtwpci, ring); 821 else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len)) 822 return -ENOSPC; 823 824 pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); 825 memset(pkt_desc, 0, tx_pkt_desc_sz); 826 pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue); 827 rtw_tx_fill_tx_desc(rtwdev, pkt_info, skb); 828 dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len, 829 DMA_TO_DEVICE); 830 if (dma_mapping_error(&rtwpci->pdev->dev, dma)) 831 return -EBUSY; 832 833 /* after this we got dma mapped, there is no way back */ 834 buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz); 835 memset(buf_desc, 0, tx_buf_desc_sz); 836 psb_len = (skb->len - 1) / 128 + 1; 837 if (queue == RTW_TX_QUEUE_BCN) 838 psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET; 839 840 buf_desc[0].psb_len = cpu_to_le16(psb_len); 841 buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz); 842 buf_desc[0].dma = cpu_to_le32(dma); 843 buf_desc[1].buf_size = cpu_to_le16(size); 844 buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz); 845 846 tx_data = rtw_pci_get_tx_data(skb); 847 tx_data->dma = dma; 848 tx_data->sn = pkt_info->sn; 849 850 spin_lock_bh(&rtwpci->irq_lock); 851 852 skb_queue_tail(&ring->queue, skb); 853 854 if (queue == RTW_TX_QUEUE_BCN) 855 goto out_unlock; 856 857 /* update write-index, and kick it off later */ 858 set_bit(queue, rtwpci->tx_queued); 859 if (++ring->r.wp >= ring->r.len) 860 ring->r.wp = 0; 861 862 out_unlock: 863 spin_unlock_bh(&rtwpci->irq_lock); 864 865 return 0; 866 } 867 868 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, 869 u32 size) 870 { 871 struct sk_buff *skb; 872 struct rtw_tx_pkt_info pkt_info = {0}; 873 u8 reg_bcn_work; 874 int ret; 875 876 skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size); 877 if (!skb) 878 return -ENOMEM; 879 880 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN); 881 if (ret) { 882 rtw_err(rtwdev, "failed to write rsvd page data\n"); 883 return ret; 884 } 885 886 /* reserved pages go through beacon queue */ 887 reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK); 888 reg_bcn_work |= BIT_PCI_BCNQ_FLAG; 889 rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work); 890 891 return 0; 892 } 893 894 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) 895 { 896 struct sk_buff *skb; 897 struct rtw_tx_pkt_info pkt_info = {0}; 898 int ret; 899 900 skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size); 901 if (!skb) 902 return -ENOMEM; 903 904 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C); 905 if (ret) { 906 rtw_err(rtwdev, "failed to write h2c data\n"); 907 return ret; 908 } 909 910 rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C); 911 912 return 0; 913 } 914 915 static int rtw_pci_tx_write(struct rtw_dev *rtwdev, 916 struct rtw_tx_pkt_info *pkt_info, 917 struct sk_buff *skb) 918 { 919 enum rtw_tx_queue_type queue = rtw_tx_queue_mapping(skb); 920 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 921 struct rtw_pci_tx_ring *ring; 922 int ret; 923 924 ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue); 925 if (ret) 926 return ret; 927 928 ring = &rtwpci->tx_rings[queue]; 929 spin_lock_bh(&rtwpci->irq_lock); 930 if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) { 931 ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb)); 932 ring->queue_stopped = true; 933 } 934 spin_unlock_bh(&rtwpci->irq_lock); 935 936 return 0; 937 } 938 939 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 940 u8 hw_queue) 941 { 942 struct ieee80211_hw *hw = rtwdev->hw; 943 struct ieee80211_tx_info *info; 944 struct rtw_pci_tx_ring *ring; 945 struct rtw_pci_tx_data *tx_data; 946 struct sk_buff *skb; 947 u32 count; 948 u32 bd_idx_addr; 949 u32 bd_idx, cur_rp, rp_idx; 950 u16 q_map; 951 952 ring = &rtwpci->tx_rings[hw_queue]; 953 954 bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue]; 955 bd_idx = rtw_read32(rtwdev, bd_idx_addr); 956 cur_rp = bd_idx >> 16; 957 cur_rp &= TRX_BD_IDX_MASK; 958 rp_idx = ring->r.rp; 959 if (cur_rp >= ring->r.rp) 960 count = cur_rp - ring->r.rp; 961 else 962 count = ring->r.len - (ring->r.rp - cur_rp); 963 964 while (count--) { 965 skb = skb_dequeue(&ring->queue); 966 if (!skb) { 967 rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n", 968 count, hw_queue, bd_idx, ring->r.rp, cur_rp); 969 break; 970 } 971 tx_data = rtw_pci_get_tx_data(skb); 972 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, 973 DMA_TO_DEVICE); 974 975 /* just free command packets from host to card */ 976 if (hw_queue == RTW_TX_QUEUE_H2C) { 977 dev_kfree_skb_irq(skb); 978 continue; 979 } 980 981 if (ring->queue_stopped && 982 avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) { 983 q_map = skb_get_queue_mapping(skb); 984 ieee80211_wake_queue(hw, q_map); 985 ring->queue_stopped = false; 986 } 987 988 if (++rp_idx >= ring->r.len) 989 rp_idx = 0; 990 991 skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz); 992 993 info = IEEE80211_SKB_CB(skb); 994 995 /* enqueue to wait for tx report */ 996 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { 997 rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); 998 continue; 999 } 1000 1001 /* always ACK for others, then they won't be marked as drop */ 1002 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 1003 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; 1004 else 1005 info->flags |= IEEE80211_TX_STAT_ACK; 1006 1007 ieee80211_tx_info_clear_status(info); 1008 ieee80211_tx_status_irqsafe(hw, skb); 1009 } 1010 1011 ring->r.rp = cur_rp; 1012 } 1013 1014 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev) 1015 { 1016 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1017 struct napi_struct *napi = &rtwpci->napi; 1018 1019 napi_schedule(napi); 1020 } 1021 1022 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev, 1023 struct rtw_pci *rtwpci) 1024 { 1025 struct rtw_pci_rx_ring *ring; 1026 int count = 0; 1027 u32 tmp, cur_wp; 1028 1029 ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1030 tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ); 1031 cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK); 1032 if (cur_wp >= ring->r.wp) 1033 count = cur_wp - ring->r.wp; 1034 else 1035 count = ring->r.len - (ring->r.wp - cur_wp); 1036 1037 return count; 1038 } 1039 1040 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 1041 u8 hw_queue, u32 limit) 1042 { 1043 const struct rtw_chip_info *chip = rtwdev->chip; 1044 struct napi_struct *napi = &rtwpci->napi; 1045 struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1046 struct rtw_rx_pkt_stat pkt_stat; 1047 struct ieee80211_rx_status rx_status; 1048 struct sk_buff *skb, *new; 1049 u32 cur_rp = ring->r.rp; 1050 u32 count, rx_done = 0; 1051 u32 pkt_offset; 1052 u32 pkt_desc_sz = chip->rx_pkt_desc_sz; 1053 u32 buf_desc_sz = chip->rx_buf_desc_sz; 1054 u32 new_len; 1055 u8 *rx_desc; 1056 dma_addr_t dma; 1057 1058 count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci); 1059 count = min(count, limit); 1060 1061 while (count--) { 1062 rtw_pci_dma_check(rtwdev, ring, cur_rp); 1063 skb = ring->buf[cur_rp]; 1064 dma = *((dma_addr_t *)skb->cb); 1065 dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE, 1066 DMA_FROM_DEVICE); 1067 rx_desc = skb->data; 1068 rtw_rx_query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status); 1069 1070 /* offset from rx_desc to payload */ 1071 pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + 1072 pkt_stat.shift; 1073 1074 /* allocate a new skb for this frame, 1075 * discard the frame if none available 1076 */ 1077 new_len = pkt_stat.pkt_len + pkt_offset; 1078 new = dev_alloc_skb(new_len); 1079 if (WARN_ONCE(!new, "rx routine starvation\n")) 1080 goto next_rp; 1081 1082 /* put the DMA data including rx_desc from phy to new skb */ 1083 skb_put_data(new, skb->data, new_len); 1084 1085 if (pkt_stat.is_c2h) { 1086 rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new); 1087 } else { 1088 /* remove rx_desc */ 1089 skb_pull(new, pkt_offset); 1090 1091 rtw_update_rx_freq_for_invalid(rtwdev, new, &rx_status, &pkt_stat); 1092 rtw_rx_stats(rtwdev, pkt_stat.vif, new); 1093 memcpy(new->cb, &rx_status, sizeof(rx_status)); 1094 ieee80211_rx_napi(rtwdev->hw, NULL, new, napi); 1095 rx_done++; 1096 } 1097 1098 next_rp: 1099 /* new skb delivered to mac80211, re-enable original skb DMA */ 1100 rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp, 1101 buf_desc_sz); 1102 1103 /* host read next element in ring */ 1104 if (++cur_rp >= ring->r.len) 1105 cur_rp = 0; 1106 } 1107 1108 ring->r.rp = cur_rp; 1109 /* 'rp', the last position we have read, is seen as previous posistion 1110 * of 'wp' that is used to calculate 'count' next time. 1111 */ 1112 ring->r.wp = cur_rp; 1113 rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp); 1114 1115 return rx_done; 1116 } 1117 1118 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev, 1119 struct rtw_pci *rtwpci, u32 *irq_status) 1120 { 1121 unsigned long flags; 1122 1123 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 1124 1125 irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0); 1126 irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1); 1127 if (rtw_chip_wcpu_11ac(rtwdev)) 1128 irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3); 1129 else 1130 irq_status[3] = 0; 1131 irq_status[0] &= rtwpci->irq_mask[0]; 1132 irq_status[1] &= rtwpci->irq_mask[1]; 1133 irq_status[3] &= rtwpci->irq_mask[3]; 1134 rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]); 1135 rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]); 1136 if (rtw_chip_wcpu_11ac(rtwdev)) 1137 rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]); 1138 1139 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 1140 } 1141 1142 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev) 1143 { 1144 struct rtw_dev *rtwdev = dev; 1145 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1146 1147 /* disable RTW PCI interrupt to avoid more interrupts before the end of 1148 * thread function 1149 * 1150 * disable HIMR here to also avoid new HISR flag being raised before 1151 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs 1152 * are cleared, the edge-triggered interrupt will not be generated when 1153 * a new HISR flag is set. 1154 */ 1155 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1156 1157 return IRQ_WAKE_THREAD; 1158 } 1159 1160 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev) 1161 { 1162 struct rtw_dev *rtwdev = dev; 1163 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1164 u32 irq_status[4]; 1165 bool rx = false; 1166 1167 spin_lock_bh(&rtwpci->irq_lock); 1168 rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status); 1169 1170 if (irq_status[0] & IMR_MGNTDOK) 1171 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT); 1172 if (irq_status[0] & IMR_HIGHDOK) 1173 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0); 1174 if (irq_status[0] & IMR_BEDOK) 1175 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE); 1176 if (irq_status[0] & IMR_BKDOK) 1177 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK); 1178 if (irq_status[0] & IMR_VODOK) 1179 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO); 1180 if (irq_status[0] & IMR_VIDOK) 1181 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI); 1182 if (irq_status[3] & IMR_H2CDOK) 1183 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C); 1184 if (irq_status[0] & IMR_ROK) { 1185 rtw_pci_rx_isr(rtwdev); 1186 rx = true; 1187 } 1188 if (unlikely(irq_status[0] & IMR_C2HCMD)) 1189 rtw_fw_c2h_cmd_isr(rtwdev); 1190 1191 /* all of the jobs for this interrupt have been done */ 1192 if (rtwpci->running) 1193 rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); 1194 spin_unlock_bh(&rtwpci->irq_lock); 1195 1196 return IRQ_HANDLED; 1197 } 1198 1199 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev, 1200 struct pci_dev *pdev) 1201 { 1202 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1203 unsigned long len; 1204 u8 bar_id = 2; 1205 int ret; 1206 1207 ret = pci_request_regions(pdev, KBUILD_MODNAME); 1208 if (ret) { 1209 rtw_err(rtwdev, "failed to request pci regions\n"); 1210 return ret; 1211 } 1212 1213 len = pci_resource_len(pdev, bar_id); 1214 rtwpci->mmap = pci_iomap(pdev, bar_id, len); 1215 if (!rtwpci->mmap) { 1216 pci_release_regions(pdev); 1217 rtw_err(rtwdev, "failed to map pci memory\n"); 1218 return -ENOMEM; 1219 } 1220 1221 return 0; 1222 } 1223 1224 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev, 1225 struct pci_dev *pdev) 1226 { 1227 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1228 1229 if (rtwpci->mmap) { 1230 pci_iounmap(pdev, rtwpci->mmap); 1231 pci_release_regions(pdev); 1232 } 1233 } 1234 1235 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data) 1236 { 1237 u16 write_addr; 1238 u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK); 1239 u8 flag; 1240 u8 cnt; 1241 1242 write_addr = addr & BITS_DBI_ADDR_MASK; 1243 write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN); 1244 rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data); 1245 rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr); 1246 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16); 1247 1248 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1249 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1250 if (flag == 0) 1251 return; 1252 1253 udelay(10); 1254 } 1255 1256 WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr); 1257 } 1258 1259 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value) 1260 { 1261 u16 read_addr = addr & BITS_DBI_ADDR_MASK; 1262 u8 flag; 1263 u8 cnt; 1264 1265 rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr); 1266 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16); 1267 1268 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1269 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1270 if (flag == 0) { 1271 read_addr = REG_DBI_RDATA_V1 + (addr & 3); 1272 *value = rtw_read8(rtwdev, read_addr); 1273 return 0; 1274 } 1275 1276 udelay(10); 1277 } 1278 1279 WARN(1, "failed to read DBI register, addr=0x%04x\n", addr); 1280 return -EIO; 1281 } 1282 1283 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1) 1284 { 1285 u8 page; 1286 u8 wflag; 1287 u8 cnt; 1288 1289 rtw_write16(rtwdev, REG_MDIO_V1, data); 1290 1291 page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1; 1292 page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2; 1293 rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK); 1294 rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page); 1295 rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1); 1296 1297 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1298 wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG, 1299 BIT_MDIO_WFLAG_V1); 1300 if (wflag == 0) 1301 return; 1302 1303 udelay(10); 1304 } 1305 1306 WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr); 1307 } 1308 1309 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable) 1310 { 1311 u8 value; 1312 int ret; 1313 1314 if (rtw_pci_disable_aspm) 1315 return; 1316 1317 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1318 if (ret) { 1319 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1320 return; 1321 } 1322 1323 if (enable) 1324 value |= BIT_CLKREQ_SW_EN; 1325 else 1326 value &= ~BIT_CLKREQ_SW_EN; 1327 1328 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1329 } 1330 1331 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable) 1332 { 1333 u8 value; 1334 int ret; 1335 1336 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1337 if (ret) { 1338 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1339 return; 1340 } 1341 1342 if (enable) 1343 value &= ~BIT_CLKREQ_N_PAD; 1344 else 1345 value |= BIT_CLKREQ_N_PAD; 1346 1347 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1348 } 1349 1350 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable) 1351 { 1352 u8 value; 1353 int ret; 1354 1355 if (rtw_pci_disable_aspm) 1356 return; 1357 1358 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1359 if (ret) { 1360 rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret); 1361 return; 1362 } 1363 1364 if (enable) 1365 value |= BIT_L1_SW_EN; 1366 else 1367 value &= ~BIT_L1_SW_EN; 1368 1369 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1370 } 1371 1372 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter) 1373 { 1374 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1375 1376 /* Like CLKREQ, ASPM is also implemented by two HW modules, and can 1377 * only be enabled when host supports it. 1378 * 1379 * And ASPM mechanism should be enabled when driver/firmware enters 1380 * power save mode, without having heavy traffic. Because we've 1381 * experienced some inter-operability issues that the link tends 1382 * to enter L1 state on the fly even when driver is having high 1383 * throughput. This is probably because the ASPM behavior slightly 1384 * varies from different SOC. 1385 */ 1386 if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)) 1387 return; 1388 1389 if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) || 1390 (!enter && atomic_inc_return(&rtwpci->link_usage) == 1)) 1391 rtw_pci_aspm_set(rtwdev, enter); 1392 } 1393 1394 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) 1395 { 1396 const struct rtw_chip_info *chip = rtwdev->chip; 1397 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1398 struct pci_dev *pdev = rtwpci->pdev; 1399 u16 link_ctrl; 1400 int ret; 1401 1402 /* RTL8822CE has enabled REFCLK auto calibration, it does not need 1403 * to add clock delay to cover the REFCLK timing gap. 1404 */ 1405 if (chip->id == RTW_CHIP_TYPE_8822C) 1406 rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0); 1407 1408 /* Though there is standard PCIE configuration space to set the 1409 * link control register, but by Realtek's design, driver should 1410 * check if host supports CLKREQ/ASPM to enable the HW module. 1411 * 1412 * These functions are implemented by two HW modules associated, 1413 * one is responsible to access PCIE configuration space to 1414 * follow the host settings, and another is in charge of doing 1415 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes 1416 * the host does not support it, and due to some reasons or wrong 1417 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device 1418 * loss if HW misbehaves on the link. 1419 * 1420 * Hence it's designed that driver should first check the PCIE 1421 * configuration space is sync'ed and enabled, then driver can turn 1422 * on the other module that is actually working on the mechanism. 1423 */ 1424 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl); 1425 if (ret) { 1426 rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret); 1427 return; 1428 } 1429 1430 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN) 1431 rtw_pci_clkreq_set(rtwdev, true); 1432 1433 rtwpci->link_ctrl = link_ctrl; 1434 } 1435 1436 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) 1437 { 1438 const struct rtw_chip_info *chip = rtwdev->chip; 1439 1440 switch (chip->id) { 1441 case RTW_CHIP_TYPE_8822C: 1442 if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D) 1443 rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG, 1444 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1); 1445 break; 1446 default: 1447 break; 1448 } 1449 } 1450 1451 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev) 1452 { 1453 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1454 const struct rtw_chip_info *chip = rtwdev->chip; 1455 struct rtw_efuse *efuse = &rtwdev->efuse; 1456 struct pci_dev *pdev = rtwpci->pdev; 1457 const struct rtw_intf_phy_para *para; 1458 u16 cut; 1459 u16 value; 1460 u16 offset; 1461 int i; 1462 int ret; 1463 1464 cut = BIT(0) << rtwdev->hal.cut_version; 1465 1466 for (i = 0; i < chip->intf_table->n_gen1_para; i++) { 1467 para = &chip->intf_table->gen1_para[i]; 1468 if (!(para->cut_mask & cut)) 1469 continue; 1470 if (para->offset == 0xffff) 1471 break; 1472 offset = para->offset; 1473 value = para->value; 1474 if (para->ip_sel == RTW_IP_SEL_PHY) 1475 rtw_mdio_write(rtwdev, offset, value, true); 1476 else 1477 rtw_dbi_write8(rtwdev, offset, value); 1478 } 1479 1480 for (i = 0; i < chip->intf_table->n_gen2_para; i++) { 1481 para = &chip->intf_table->gen2_para[i]; 1482 if (!(para->cut_mask & cut)) 1483 continue; 1484 if (para->offset == 0xffff) 1485 break; 1486 offset = para->offset; 1487 value = para->value; 1488 if (para->ip_sel == RTW_IP_SEL_PHY) 1489 rtw_mdio_write(rtwdev, offset, value, false); 1490 else 1491 rtw_dbi_write8(rtwdev, offset, value); 1492 } 1493 1494 rtw_pci_link_cfg(rtwdev); 1495 1496 /* Disable 8821ce completion timeout by default */ 1497 if (chip->id == RTW_CHIP_TYPE_8821C) { 1498 ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, 1499 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); 1500 if (ret) 1501 rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n", 1502 ret); 1503 } 1504 1505 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 5) 1506 rtw_write32_mask(rtwdev, REG_ANAPARSW_MAC_0, BIT_CF_L_V2, 0x1); 1507 } 1508 1509 static int __maybe_unused rtw_pci_suspend(struct device *dev) 1510 { 1511 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1512 struct rtw_dev *rtwdev = hw->priv; 1513 const struct rtw_chip_info *chip = rtwdev->chip; 1514 struct rtw_efuse *efuse = &rtwdev->efuse; 1515 1516 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1517 rtw_pci_clkreq_pad_low(rtwdev, true); 1518 return 0; 1519 } 1520 1521 static int __maybe_unused rtw_pci_resume(struct device *dev) 1522 { 1523 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1524 struct rtw_dev *rtwdev = hw->priv; 1525 const struct rtw_chip_info *chip = rtwdev->chip; 1526 struct rtw_efuse *efuse = &rtwdev->efuse; 1527 1528 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1529 rtw_pci_clkreq_pad_low(rtwdev, false); 1530 return 0; 1531 } 1532 1533 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume); 1534 EXPORT_SYMBOL(rtw_pm_ops); 1535 1536 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1537 { 1538 int ret; 1539 1540 ret = pci_enable_device(pdev); 1541 if (ret) { 1542 rtw_err(rtwdev, "failed to enable pci device\n"); 1543 return ret; 1544 } 1545 1546 pci_set_master(pdev); 1547 pci_set_drvdata(pdev, rtwdev->hw); 1548 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); 1549 1550 return 0; 1551 } 1552 1553 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1554 { 1555 pci_disable_device(pdev); 1556 } 1557 1558 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1559 { 1560 struct rtw_pci *rtwpci; 1561 int ret; 1562 1563 rtwpci = (struct rtw_pci *)rtwdev->priv; 1564 rtwpci->pdev = pdev; 1565 1566 /* after this driver can access to hw registers */ 1567 ret = rtw_pci_io_mapping(rtwdev, pdev); 1568 if (ret) { 1569 rtw_err(rtwdev, "failed to request pci io region\n"); 1570 goto err_out; 1571 } 1572 1573 ret = rtw_pci_init(rtwdev); 1574 if (ret) { 1575 rtw_err(rtwdev, "failed to allocate pci resources\n"); 1576 goto err_io_unmap; 1577 } 1578 1579 return 0; 1580 1581 err_io_unmap: 1582 rtw_pci_io_unmapping(rtwdev, pdev); 1583 1584 err_out: 1585 return ret; 1586 } 1587 1588 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1589 { 1590 rtw_pci_deinit(rtwdev); 1591 rtw_pci_io_unmapping(rtwdev, pdev); 1592 } 1593 1594 static struct rtw_hci_ops rtw_pci_ops = { 1595 .tx_write = rtw_pci_tx_write, 1596 .tx_kick_off = rtw_pci_tx_kick_off, 1597 .flush_queues = rtw_pci_flush_queues, 1598 .setup = rtw_pci_setup, 1599 .start = rtw_pci_start, 1600 .stop = rtw_pci_stop, 1601 .deep_ps = rtw_pci_deep_ps, 1602 .link_ps = rtw_pci_link_ps, 1603 .interface_cfg = rtw_pci_interface_cfg, 1604 .dynamic_rx_agg = NULL, 1605 1606 .read8 = rtw_pci_read8, 1607 .read16 = rtw_pci_read16, 1608 .read32 = rtw_pci_read32, 1609 .write8 = rtw_pci_write8, 1610 .write16 = rtw_pci_write16, 1611 .write32 = rtw_pci_write32, 1612 .write_data_rsvd_page = rtw_pci_write_data_rsvd_page, 1613 .write_data_h2c = rtw_pci_write_data_h2c, 1614 }; 1615 1616 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1617 { 1618 unsigned int flags = PCI_IRQ_INTX; 1619 int ret; 1620 1621 if (!rtw_disable_msi) 1622 flags |= PCI_IRQ_MSI; 1623 1624 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); 1625 if (ret < 0) { 1626 rtw_err(rtwdev, "failed to alloc PCI irq vectors\n"); 1627 return ret; 1628 } 1629 1630 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, 1631 rtw_pci_interrupt_handler, 1632 rtw_pci_interrupt_threadfn, 1633 IRQF_SHARED, KBUILD_MODNAME, rtwdev); 1634 if (ret) { 1635 rtw_err(rtwdev, "failed to request irq %d\n", ret); 1636 pci_free_irq_vectors(pdev); 1637 } 1638 1639 return ret; 1640 } 1641 1642 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1643 { 1644 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); 1645 pci_free_irq_vectors(pdev); 1646 } 1647 1648 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget) 1649 { 1650 struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi); 1651 struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev, 1652 priv); 1653 int work_done = 0; 1654 1655 if (rtwpci->rx_no_aspm) 1656 rtw_pci_link_ps(rtwdev, false); 1657 1658 while (work_done < budget) { 1659 u32 work_done_once; 1660 1661 work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU, 1662 budget - work_done); 1663 if (work_done_once == 0) 1664 break; 1665 work_done += work_done_once; 1666 } 1667 if (work_done < budget) { 1668 napi_complete_done(napi, work_done); 1669 spin_lock_bh(&rtwpci->irq_lock); 1670 if (rtwpci->running) 1671 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 1672 spin_unlock_bh(&rtwpci->irq_lock); 1673 /* When ISR happens during polling and before napi_complete 1674 * while no further data is received. Data on the dma_ring will 1675 * not be processed immediately. Check whether dma ring is 1676 * empty and perform napi_schedule accordingly. 1677 */ 1678 if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci)) 1679 napi_schedule(napi); 1680 } 1681 if (rtwpci->rx_no_aspm) 1682 rtw_pci_link_ps(rtwdev, true); 1683 1684 return work_done; 1685 } 1686 1687 static int rtw_pci_napi_init(struct rtw_dev *rtwdev) 1688 { 1689 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1690 1691 rtwpci->netdev = alloc_netdev_dummy(0); 1692 if (!rtwpci->netdev) 1693 return -ENOMEM; 1694 1695 netif_napi_add(rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll); 1696 return 0; 1697 } 1698 1699 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) 1700 { 1701 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1702 1703 rtw_pci_napi_stop(rtwdev); 1704 netif_napi_del(&rtwpci->napi); 1705 free_netdev(rtwpci->netdev); 1706 } 1707 1708 int rtw_pci_probe(struct pci_dev *pdev, 1709 const struct pci_device_id *id) 1710 { 1711 struct pci_dev *bridge = pci_upstream_bridge(pdev); 1712 struct ieee80211_hw *hw; 1713 struct rtw_dev *rtwdev; 1714 struct rtw_pci *rtwpci; 1715 int drv_data_size; 1716 int ret; 1717 1718 drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci); 1719 hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); 1720 if (!hw) { 1721 dev_err(&pdev->dev, "failed to allocate hw\n"); 1722 return -ENOMEM; 1723 } 1724 1725 rtwdev = hw->priv; 1726 rtwdev->hw = hw; 1727 rtwdev->dev = &pdev->dev; 1728 rtwdev->chip = (struct rtw_chip_info *)id->driver_data; 1729 rtwdev->hci.ops = &rtw_pci_ops; 1730 rtwdev->hci.type = RTW_HCI_TYPE_PCIE; 1731 1732 rtwpci = (struct rtw_pci *)rtwdev->priv; 1733 atomic_set(&rtwpci->link_usage, 1); 1734 1735 ret = rtw_core_init(rtwdev); 1736 if (ret) 1737 goto err_release_hw; 1738 1739 rtw_dbg(rtwdev, RTW_DBG_PCI, 1740 "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n", 1741 pdev->vendor, pdev->device, pdev->revision); 1742 1743 ret = rtw_pci_claim(rtwdev, pdev); 1744 if (ret) { 1745 rtw_err(rtwdev, "failed to claim pci device\n"); 1746 goto err_deinit_core; 1747 } 1748 1749 ret = rtw_pci_setup_resource(rtwdev, pdev); 1750 if (ret) { 1751 rtw_err(rtwdev, "failed to setup pci resources\n"); 1752 goto err_pci_declaim; 1753 } 1754 1755 ret = rtw_pci_napi_init(rtwdev); 1756 if (ret) { 1757 rtw_err(rtwdev, "failed to setup NAPI\n"); 1758 goto err_pci_declaim; 1759 } 1760 1761 ret = rtw_chip_info_setup(rtwdev); 1762 if (ret) { 1763 rtw_err(rtwdev, "failed to setup chip information\n"); 1764 goto err_destroy_pci; 1765 } 1766 1767 /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */ 1768 if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL) 1769 rtwpci->rx_no_aspm = true; 1770 1771 rtw_pci_phy_cfg(rtwdev); 1772 1773 ret = rtw_register_hw(rtwdev, hw); 1774 if (ret) { 1775 rtw_err(rtwdev, "failed to register hw\n"); 1776 goto err_destroy_pci; 1777 } 1778 1779 ret = rtw_pci_request_irq(rtwdev, pdev); 1780 if (ret) { 1781 ieee80211_unregister_hw(hw); 1782 goto err_destroy_pci; 1783 } 1784 1785 return 0; 1786 1787 err_destroy_pci: 1788 rtw_pci_napi_deinit(rtwdev); 1789 rtw_pci_destroy(rtwdev, pdev); 1790 1791 err_pci_declaim: 1792 rtw_pci_declaim(rtwdev, pdev); 1793 1794 err_deinit_core: 1795 rtw_core_deinit(rtwdev); 1796 1797 err_release_hw: 1798 ieee80211_free_hw(hw); 1799 1800 return ret; 1801 } 1802 EXPORT_SYMBOL(rtw_pci_probe); 1803 1804 void rtw_pci_remove(struct pci_dev *pdev) 1805 { 1806 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1807 struct rtw_dev *rtwdev; 1808 struct rtw_pci *rtwpci; 1809 1810 if (!hw) 1811 return; 1812 1813 rtwdev = hw->priv; 1814 rtwpci = (struct rtw_pci *)rtwdev->priv; 1815 1816 rtw_unregister_hw(rtwdev, hw); 1817 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1818 rtw_pci_napi_deinit(rtwdev); 1819 rtw_pci_destroy(rtwdev, pdev); 1820 rtw_pci_declaim(rtwdev, pdev); 1821 rtw_pci_free_irq(rtwdev, pdev); 1822 rtw_core_deinit(rtwdev); 1823 ieee80211_free_hw(hw); 1824 } 1825 EXPORT_SYMBOL(rtw_pci_remove); 1826 1827 void rtw_pci_shutdown(struct pci_dev *pdev) 1828 { 1829 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1830 struct rtw_dev *rtwdev; 1831 const struct rtw_chip_info *chip; 1832 1833 if (!hw) 1834 return; 1835 1836 rtwdev = hw->priv; 1837 chip = rtwdev->chip; 1838 1839 if (chip->ops->shutdown) 1840 chip->ops->shutdown(rtwdev); 1841 1842 pci_set_power_state(pdev, PCI_D3hot); 1843 } 1844 EXPORT_SYMBOL(rtw_pci_shutdown); 1845 1846 MODULE_AUTHOR("Realtek Corporation"); 1847 MODULE_DESCRIPTION("Realtek PCI 802.11ac wireless driver"); 1848 MODULE_LICENSE("Dual BSD/GPL"); 1849