1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include <linux/dma-mapping.h> 7 #include "mt76.h" 8 #include "dma.h" 9 10 static struct mt76_txwi_cache * 11 mt76_alloc_txwi(struct mt76_dev *dev) 12 { 13 struct mt76_txwi_cache *t; 14 dma_addr_t addr; 15 u8 *txwi; 16 int size; 17 18 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t)); 19 txwi = kzalloc(size, GFP_ATOMIC); 20 if (!txwi) 21 return NULL; 22 23 addr = dma_map_single(dev->dma_dev, txwi, dev->drv->txwi_size, 24 DMA_TO_DEVICE); 25 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) { 26 kfree(txwi); 27 return NULL; 28 } 29 30 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size); 31 t->dma_addr = addr; 32 33 return t; 34 } 35 36 static struct mt76_txwi_cache * 37 mt76_alloc_rxwi(struct mt76_dev *dev) 38 { 39 struct mt76_txwi_cache *t; 40 41 t = kzalloc(L1_CACHE_ALIGN(sizeof(*t)), GFP_ATOMIC); 42 if (!t) 43 return NULL; 44 45 t->ptr = NULL; 46 return t; 47 } 48 49 static struct mt76_txwi_cache * 50 __mt76_get_txwi(struct mt76_dev *dev) 51 { 52 struct mt76_txwi_cache *t = NULL; 53 54 spin_lock(&dev->lock); 55 if (!list_empty(&dev->txwi_cache)) { 56 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache, 57 list); 58 list_del(&t->list); 59 } 60 spin_unlock(&dev->lock); 61 62 return t; 63 } 64 65 static struct mt76_txwi_cache * 66 __mt76_get_rxwi(struct mt76_dev *dev) 67 { 68 struct mt76_txwi_cache *t = NULL; 69 70 spin_lock_bh(&dev->wed_lock); 71 if (!list_empty(&dev->rxwi_cache)) { 72 t = list_first_entry(&dev->rxwi_cache, struct mt76_txwi_cache, 73 list); 74 list_del(&t->list); 75 } 76 spin_unlock_bh(&dev->wed_lock); 77 78 return t; 79 } 80 81 static struct mt76_txwi_cache * 82 mt76_get_txwi(struct mt76_dev *dev) 83 { 84 struct mt76_txwi_cache *t = __mt76_get_txwi(dev); 85 86 if (t) 87 return t; 88 89 return mt76_alloc_txwi(dev); 90 } 91 92 struct mt76_txwi_cache * 93 mt76_get_rxwi(struct mt76_dev *dev) 94 { 95 struct mt76_txwi_cache *t = __mt76_get_rxwi(dev); 96 97 if (t) 98 return t; 99 100 return mt76_alloc_rxwi(dev); 101 } 102 EXPORT_SYMBOL_GPL(mt76_get_rxwi); 103 104 void 105 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) 106 { 107 if (!t) 108 return; 109 110 spin_lock(&dev->lock); 111 list_add(&t->list, &dev->txwi_cache); 112 spin_unlock(&dev->lock); 113 } 114 EXPORT_SYMBOL_GPL(mt76_put_txwi); 115 116 void 117 mt76_put_rxwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) 118 { 119 if (!t) 120 return; 121 122 spin_lock_bh(&dev->wed_lock); 123 list_add(&t->list, &dev->rxwi_cache); 124 spin_unlock_bh(&dev->wed_lock); 125 } 126 EXPORT_SYMBOL_GPL(mt76_put_rxwi); 127 128 static void 129 mt76_free_pending_txwi(struct mt76_dev *dev) 130 { 131 struct mt76_txwi_cache *t; 132 133 local_bh_disable(); 134 while ((t = __mt76_get_txwi(dev)) != NULL) { 135 dma_unmap_single(dev->dma_dev, t->dma_addr, dev->drv->txwi_size, 136 DMA_TO_DEVICE); 137 kfree(mt76_get_txwi_ptr(dev, t)); 138 } 139 local_bh_enable(); 140 } 141 142 void 143 mt76_free_pending_rxwi(struct mt76_dev *dev) 144 { 145 struct mt76_txwi_cache *t; 146 147 local_bh_disable(); 148 while ((t = __mt76_get_rxwi(dev)) != NULL) { 149 if (t->ptr) 150 mt76_put_page_pool_buf(t->ptr, false); 151 kfree(t); 152 } 153 local_bh_enable(); 154 } 155 EXPORT_SYMBOL_GPL(mt76_free_pending_rxwi); 156 157 static void 158 mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q) 159 { 160 if (!mt76_queue_is_wed_rro(q)) 161 return; 162 163 q->magic_cnt = 0; 164 if (mt76_queue_is_wed_rro_ind(q)) { 165 struct mt76_wed_rro_desc *rro_desc; 166 u32 data1 = FIELD_PREP(RRO_IND_DATA1_MAGIC_CNT_MASK, 167 MT_DMA_WED_IND_CMD_CNT - 1); 168 int i; 169 170 rro_desc = (struct mt76_wed_rro_desc *)q->desc; 171 for (i = 0; i < q->ndesc; i++) { 172 struct mt76_wed_rro_ind *cmd; 173 174 cmd = (struct mt76_wed_rro_ind *)&rro_desc[i]; 175 cmd->data1 = cpu_to_le32(data1); 176 } 177 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { 178 struct mt76_rro_rxdmad_c *dmad = (void *)q->desc; 179 u32 data3 = FIELD_PREP(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK, 180 MT_DMA_MAGIC_CNT - 1); 181 int i; 182 183 for (i = 0; i < q->ndesc; i++) 184 dmad[i].data3 = cpu_to_le32(data3); 185 } 186 } 187 188 static void 189 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) 190 { 191 Q_WRITE(q, desc_base, q->desc_dma); 192 if ((q->flags & MT_QFLAG_WED_RRO_EN) && !mt76_npu_device_active(dev)) 193 Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc); 194 else 195 Q_WRITE(q, ring_size, q->ndesc); 196 197 if (mt76_queue_is_npu_tx(q)) { 198 writel(q->desc_dma, &q->regs->desc_base); 199 writel(q->ndesc, &q->regs->ring_size); 200 } 201 q->head = Q_READ(q, dma_idx); 202 q->tail = q->head; 203 } 204 205 void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, 206 bool reset_idx) 207 { 208 if (!q || !q->ndesc) 209 return; 210 211 if (!mt76_queue_is_wed_rro_ind(q) && 212 !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { 213 int i; 214 215 /* clear descriptors */ 216 for (i = 0; i < q->ndesc; i++) 217 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); 218 } 219 220 mt76_dma_queue_magic_cnt_init(dev, q); 221 if (reset_idx) { 222 if (mt76_queue_is_emi(q)) 223 *q->emi_cpu_idx = 0; 224 else 225 Q_WRITE(q, cpu_idx, 0); 226 Q_WRITE(q, dma_idx, 0); 227 } 228 mt76_dma_sync_idx(dev, q); 229 } 230 231 static int 232 mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, 233 struct mt76_queue_buf *buf, void *data) 234 { 235 struct mt76_queue_entry *entry = &q->entry[q->head]; 236 struct mt76_txwi_cache *txwi = NULL; 237 u32 buf1 = 0, ctrl, info = 0; 238 struct mt76_desc *desc; 239 int idx = q->head; 240 int rx_token; 241 242 if (mt76_queue_is_wed_rro_ind(q)) { 243 struct mt76_wed_rro_desc *rro_desc; 244 245 rro_desc = (struct mt76_wed_rro_desc *)q->desc; 246 data = &rro_desc[q->head]; 247 goto done; 248 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { 249 data = &q->desc[q->head]; 250 goto done; 251 } 252 253 desc = &q->desc[q->head]; 254 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); 255 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 256 buf1 = FIELD_PREP(MT_DMA_CTL_SDP0_H, buf->addr >> 32); 257 #endif 258 259 if (mt76_queue_is_wed_rx(q) || mt76_queue_is_wed_rro_data(q)) { 260 txwi = mt76_get_rxwi(dev); 261 if (!txwi) 262 return -ENOMEM; 263 264 rx_token = mt76_rx_token_consume(dev, data, txwi, buf->addr); 265 if (rx_token < 0) { 266 mt76_put_rxwi(dev, txwi); 267 return -ENOMEM; 268 } 269 270 buf1 |= FIELD_PREP(MT_DMA_CTL_TOKEN, rx_token); 271 ctrl |= MT_DMA_CTL_TO_HOST; 272 273 txwi->qid = q - dev->q_rx; 274 } 275 276 if (mt76_queue_is_wed_rro_msdu_pg(q) && 277 dev->drv->rx_rro_add_msdu_page) { 278 if (dev->drv->rx_rro_add_msdu_page(dev, q, buf->addr, data)) 279 return -ENOMEM; 280 } 281 282 if (q->flags & MT_QFLAG_WED_RRO_EN) { 283 info |= FIELD_PREP(MT_DMA_MAGIC_MASK, q->magic_cnt); 284 if ((q->head + 1) == q->ndesc) 285 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT; 286 } 287 288 WRITE_ONCE(desc->buf0, cpu_to_le32(buf->addr)); 289 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); 290 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); 291 WRITE_ONCE(desc->info, cpu_to_le32(info)); 292 293 done: 294 entry->dma_addr[0] = buf->addr; 295 entry->dma_len[0] = buf->len; 296 entry->txwi = txwi; 297 entry->buf = data; 298 entry->wcid = 0xffff; 299 entry->skip_buf1 = true; 300 q->head = (q->head + 1) % q->ndesc; 301 q->queued++; 302 303 return idx; 304 } 305 306 static int 307 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, 308 struct mt76_queue_buf *buf, int nbufs, u32 info, 309 struct sk_buff *skb, void *txwi) 310 { 311 struct mt76_queue_entry *entry; 312 struct mt76_desc *desc; 313 int i, idx = -1; 314 u32 ctrl, next; 315 316 if (txwi) { 317 q->entry[q->head].txwi = DMA_DUMMY_DATA; 318 q->entry[q->head].skip_buf0 = true; 319 } 320 321 for (i = 0; i < nbufs; i += 2, buf += 2) { 322 u32 buf0 = buf[0].addr, buf1 = 0; 323 324 idx = q->head; 325 next = (q->head + 1) % q->ndesc; 326 327 desc = &q->desc[idx]; 328 entry = &q->entry[idx]; 329 330 if (buf[0].skip_unmap) 331 entry->skip_buf0 = true; 332 entry->skip_buf1 = i == nbufs - 1; 333 334 entry->dma_addr[0] = buf[0].addr; 335 entry->dma_len[0] = buf[0].len; 336 337 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); 338 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 339 info |= FIELD_PREP(MT_DMA_CTL_SDP0_H, buf[0].addr >> 32); 340 #endif 341 if (i < nbufs - 1) { 342 entry->dma_addr[1] = buf[1].addr; 343 entry->dma_len[1] = buf[1].len; 344 buf1 = buf[1].addr; 345 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len); 346 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 347 info |= FIELD_PREP(MT_DMA_CTL_SDP1_H, 348 buf[1].addr >> 32); 349 #endif 350 if (buf[1].skip_unmap) 351 entry->skip_buf1 = true; 352 } 353 354 if (i == nbufs - 1) 355 ctrl |= MT_DMA_CTL_LAST_SEC0; 356 else if (i == nbufs - 2) 357 ctrl |= MT_DMA_CTL_LAST_SEC1; 358 359 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0)); 360 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); 361 WRITE_ONCE(desc->info, cpu_to_le32(info)); 362 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); 363 364 q->head = next; 365 q->queued++; 366 } 367 368 q->entry[idx].txwi = txwi; 369 q->entry[idx].skb = skb; 370 q->entry[idx].wcid = 0xffff; 371 372 return idx; 373 } 374 375 static void 376 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx, 377 struct mt76_queue_entry *prev_e) 378 { 379 struct mt76_queue_entry *e = &q->entry[idx]; 380 381 if (!e->skip_buf0) 382 dma_unmap_single(dev->dma_dev, e->dma_addr[0], e->dma_len[0], 383 DMA_TO_DEVICE); 384 385 if (!e->skip_buf1) 386 dma_unmap_single(dev->dma_dev, e->dma_addr[1], e->dma_len[1], 387 DMA_TO_DEVICE); 388 389 if (e->txwi == DMA_DUMMY_DATA) 390 e->txwi = NULL; 391 392 *prev_e = *e; 393 memset(e, 0, sizeof(*e)); 394 } 395 396 static void 397 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q) 398 { 399 wmb(); 400 if (mt76_queue_is_emi(q)) 401 *q->emi_cpu_idx = cpu_to_le16(q->head); 402 else 403 Q_WRITE(q, cpu_idx, q->head); 404 } 405 406 static void 407 mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) 408 { 409 struct mt76_queue_entry entry; 410 int last; 411 412 if (!q || !q->ndesc) 413 return; 414 415 spin_lock_bh(&q->cleanup_lock); 416 if (flush) 417 last = -1; 418 else 419 last = Q_READ(q, dma_idx); 420 421 while (q->queued > 0 && q->tail != last) { 422 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); 423 mt76_npu_txdesc_cleanup(q, q->tail); 424 mt76_queue_tx_complete(dev, q, &entry); 425 426 if (entry.txwi) { 427 if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE)) 428 mt76_put_txwi(dev, entry.txwi); 429 } 430 431 if (!flush && q->tail == last) 432 last = Q_READ(q, dma_idx); 433 } 434 spin_unlock_bh(&q->cleanup_lock); 435 436 if (flush) { 437 spin_lock_bh(&q->lock); 438 mt76_dma_sync_idx(dev, q); 439 mt76_dma_kick_queue(dev, q); 440 spin_unlock_bh(&q->lock); 441 } 442 443 if (!q->queued) 444 wake_up(&dev->tx_wait); 445 } 446 447 static void * 448 mt76_dma_get_rxdmad_c_buf(struct mt76_dev *dev, struct mt76_queue *q, 449 int idx, int *len, bool *more) 450 { 451 struct mt76_queue_entry *e = &q->entry[idx]; 452 struct mt76_rro_rxdmad_c *dmad = e->buf; 453 u32 data1 = le32_to_cpu(dmad->data1); 454 u32 data2 = le32_to_cpu(dmad->data2); 455 struct mt76_txwi_cache *t; 456 u16 rx_token_id; 457 u8 ind_reason; 458 void *buf; 459 460 rx_token_id = FIELD_GET(RRO_RXDMAD_DATA2_RX_TOKEN_ID_MASK, data2); 461 t = mt76_rx_token_release(dev, rx_token_id); 462 if (!t) 463 return ERR_PTR(-EAGAIN); 464 465 q = &dev->q_rx[t->qid]; 466 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, 467 SKB_WITH_OVERHEAD(q->buf_size), 468 page_pool_get_dma_dir(q->page_pool)); 469 470 if (len) 471 *len = FIELD_GET(RRO_RXDMAD_DATA1_SDL0_MASK, data1); 472 if (more) 473 *more = !FIELD_GET(RRO_RXDMAD_DATA1_LS_MASK, data1); 474 475 buf = t->ptr; 476 ind_reason = FIELD_GET(RRO_RXDMAD_DATA2_IND_REASON_MASK, data2); 477 if (ind_reason == MT_DMA_WED_IND_REASON_REPEAT || 478 ind_reason == MT_DMA_WED_IND_REASON_OLDPKT) { 479 mt76_put_page_pool_buf(buf, false); 480 buf = ERR_PTR(-EAGAIN); 481 } 482 t->ptr = NULL; 483 t->dma_addr = 0; 484 485 mt76_put_rxwi(dev, t); 486 487 return buf; 488 } 489 490 static void * 491 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, 492 int *len, u32 *info, bool *more, bool *drop, bool flush) 493 { 494 struct mt76_queue_entry *e = &q->entry[idx]; 495 struct mt76_desc *desc = &q->desc[idx]; 496 u32 ctrl, desc_info, buf1; 497 void *buf = e->buf; 498 499 if (mt76_queue_is_wed_rro_rxdmad_c(q) && !flush) 500 buf = mt76_dma_get_rxdmad_c_buf(dev, q, idx, len, more); 501 502 if (mt76_queue_is_wed_rro(q)) 503 goto done; 504 505 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); 506 if (len) { 507 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl); 508 *more = !(ctrl & MT_DMA_CTL_LAST_SEC0); 509 } 510 511 desc_info = le32_to_cpu(desc->info); 512 if (info) 513 *info = desc_info; 514 515 buf1 = le32_to_cpu(desc->buf1); 516 mt76_dma_should_drop_buf(drop, ctrl, buf1, desc_info); 517 518 if (mt76_queue_is_wed_rx(q)) { 519 u32 token = FIELD_GET(MT_DMA_CTL_TOKEN, buf1); 520 struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token); 521 522 if (!t) 523 return NULL; 524 525 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, 526 SKB_WITH_OVERHEAD(q->buf_size), 527 page_pool_get_dma_dir(q->page_pool)); 528 529 buf = t->ptr; 530 t->dma_addr = 0; 531 t->ptr = NULL; 532 533 mt76_put_rxwi(dev, t); 534 if (drop) 535 *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP); 536 } else { 537 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], 538 SKB_WITH_OVERHEAD(q->buf_size), 539 page_pool_get_dma_dir(q->page_pool)); 540 } 541 542 done: 543 e->buf = NULL; 544 return buf; 545 } 546 547 static void * 548 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, 549 int *len, u32 *info, bool *more, bool *drop) 550 { 551 int idx = q->tail; 552 553 *more = false; 554 if (!q->queued) 555 return NULL; 556 557 if (mt76_queue_is_wed_rro_data(q) || mt76_queue_is_wed_rro_msdu_pg(q)) 558 goto done; 559 560 if (mt76_queue_is_wed_rro_ind(q)) { 561 struct mt76_wed_rro_ind *cmd; 562 u8 magic_cnt; 563 564 if (flush) 565 goto done; 566 567 cmd = q->entry[idx].buf; 568 magic_cnt = FIELD_GET(RRO_IND_DATA1_MAGIC_CNT_MASK, 569 le32_to_cpu(cmd->data1)); 570 if (magic_cnt != q->magic_cnt) 571 return NULL; 572 573 if (q->tail == q->ndesc - 1) 574 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_WED_IND_CMD_CNT; 575 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { 576 struct mt76_rro_rxdmad_c *dmad; 577 u16 magic_cnt; 578 579 if (flush) 580 goto done; 581 582 dmad = q->entry[idx].buf; 583 magic_cnt = FIELD_GET(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK, 584 le32_to_cpu(dmad->data3)); 585 if (magic_cnt != q->magic_cnt) 586 return NULL; 587 588 if (q->tail == q->ndesc - 1) 589 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT; 590 } else { 591 if (flush) 592 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); 593 else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) 594 return NULL; 595 } 596 done: 597 q->tail = (q->tail + 1) % q->ndesc; 598 q->queued--; 599 600 return mt76_dma_get_buf(dev, q, idx, len, info, more, drop, flush); 601 } 602 603 static int 604 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q, 605 struct sk_buff *skb, u32 tx_info) 606 { 607 struct mt76_queue_buf buf = {}; 608 dma_addr_t addr; 609 610 if (test_bit(MT76_MCU_RESET, &dev->phy.state)) 611 goto error; 612 613 if (q->queued + 1 >= q->ndesc - 1) 614 goto error; 615 616 addr = dma_map_single(dev->dma_dev, skb->data, skb->len, 617 DMA_TO_DEVICE); 618 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) 619 goto error; 620 621 buf.addr = addr; 622 buf.len = skb->len; 623 624 spin_lock_bh(&q->lock); 625 mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL); 626 mt76_dma_kick_queue(dev, q); 627 spin_unlock_bh(&q->lock); 628 629 return 0; 630 631 error: 632 dev_kfree_skb(skb); 633 return -ENOMEM; 634 } 635 636 static int 637 mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, 638 enum mt76_txq_id qid, struct sk_buff *skb, 639 struct mt76_wcid *wcid, struct ieee80211_sta *sta) 640 { 641 struct ieee80211_tx_status status = { 642 .sta = sta, 643 }; 644 struct mt76_tx_info tx_info = { 645 .skb = skb, 646 }; 647 struct mt76_dev *dev = phy->dev; 648 struct ieee80211_hw *hw; 649 int len, n = 0, ret = -ENOMEM; 650 struct mt76_txwi_cache *t; 651 struct sk_buff *iter; 652 dma_addr_t addr; 653 u8 *txwi; 654 655 if (test_bit(MT76_RESET, &phy->state)) 656 goto free_skb; 657 658 /* TODO: Take into account unlinear skbs */ 659 if (mt76_npu_device_active(dev) && skb_linearize(skb)) 660 goto free_skb; 661 662 t = mt76_get_txwi(dev); 663 if (!t) 664 goto free_skb; 665 666 txwi = mt76_get_txwi_ptr(dev, t); 667 668 skb->prev = skb->next = NULL; 669 if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS) 670 mt76_insert_hdr_pad(skb); 671 672 len = skb_headlen(skb); 673 addr = dma_map_single(dev->dma_dev, skb->data, len, DMA_TO_DEVICE); 674 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) 675 goto free; 676 677 tx_info.buf[n].addr = t->dma_addr; 678 tx_info.buf[n++].len = dev->drv->txwi_size; 679 tx_info.buf[n].addr = addr; 680 tx_info.buf[n++].len = len; 681 682 skb_walk_frags(skb, iter) { 683 if (n == ARRAY_SIZE(tx_info.buf)) 684 goto unmap; 685 686 addr = dma_map_single(dev->dma_dev, iter->data, iter->len, 687 DMA_TO_DEVICE); 688 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) 689 goto unmap; 690 691 tx_info.buf[n].addr = addr; 692 tx_info.buf[n++].len = iter->len; 693 } 694 tx_info.nbuf = n; 695 696 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) { 697 ret = -ENOMEM; 698 goto unmap; 699 } 700 701 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, dev->drv->txwi_size, 702 DMA_TO_DEVICE); 703 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info); 704 dma_sync_single_for_device(dev->dma_dev, t->dma_addr, dev->drv->txwi_size, 705 DMA_TO_DEVICE); 706 if (ret < 0) 707 goto unmap; 708 709 if (mt76_npu_device_active(dev)) 710 return mt76_npu_dma_add_buf(phy, q, skb, &tx_info.buf[1], txwi); 711 712 return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf, 713 tx_info.info, tx_info.skb, t); 714 715 unmap: 716 for (n--; n > 0; n--) 717 dma_unmap_single(dev->dma_dev, tx_info.buf[n].addr, 718 tx_info.buf[n].len, DMA_TO_DEVICE); 719 720 free: 721 #ifdef CONFIG_NL80211_TESTMODE 722 /* fix tx_done accounting on queue overflow */ 723 if (mt76_is_testmode_skb(dev, skb, &hw)) { 724 struct mt76_phy *phy = hw->priv; 725 726 if (tx_info.skb == phy->test.tx_skb) 727 phy->test.tx_done--; 728 } 729 #endif 730 731 mt76_put_txwi(dev, t); 732 733 free_skb: 734 status.skb = tx_info.skb; 735 hw = mt76_tx_status_get_hw(dev, tx_info.skb); 736 spin_lock_bh(&dev->rx_lock); 737 ieee80211_tx_status_ext(hw, &status); 738 spin_unlock_bh(&dev->rx_lock); 739 740 return ret; 741 } 742 743 static int 744 mt76_dma_rx_fill_buf(struct mt76_dev *dev, struct mt76_queue *q, 745 bool allow_direct) 746 { 747 int len = SKB_WITH_OVERHEAD(q->buf_size); 748 int frames = 0; 749 750 if (!q->ndesc) 751 return 0; 752 753 while (q->queued < q->ndesc - 1) { 754 struct mt76_queue_buf qbuf = {}; 755 void *buf = NULL; 756 int offset; 757 758 if (mt76_queue_is_wed_rro_ind(q) || 759 mt76_queue_is_wed_rro_rxdmad_c(q)) 760 goto done; 761 762 buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); 763 if (!buf) 764 break; 765 766 qbuf.addr = page_pool_get_dma_addr(virt_to_head_page(buf)) + 767 offset + q->buf_offset; 768 done: 769 qbuf.len = len - q->buf_offset; 770 qbuf.skip_unmap = false; 771 if (mt76_dma_add_rx_buf(dev, q, &qbuf, buf) < 0) { 772 mt76_put_page_pool_buf(buf, allow_direct); 773 break; 774 } 775 frames++; 776 } 777 778 if (frames || mt76_queue_is_wed_rx(q)) 779 mt76_dma_kick_queue(dev, q); 780 781 return frames; 782 } 783 784 int mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, 785 bool allow_direct) 786 { 787 int frames; 788 789 spin_lock_bh(&q->lock); 790 frames = mt76_dma_rx_fill_buf(dev, q, allow_direct); 791 spin_unlock_bh(&q->lock); 792 793 return frames; 794 } 795 796 static int 797 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, 798 int idx, int n_desc, int bufsize, 799 u32 ring_base) 800 { 801 int ret, size; 802 803 spin_lock_init(&q->lock); 804 spin_lock_init(&q->cleanup_lock); 805 806 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE; 807 q->ndesc = n_desc; 808 q->buf_size = bufsize; 809 q->hw_idx = idx; 810 q->dev = dev; 811 812 if (mt76_queue_is_wed_rro_ind(q)) 813 size = sizeof(struct mt76_wed_rro_desc); 814 else if (mt76_queue_is_npu_tx(q)) 815 size = sizeof(struct airoha_npu_tx_dma_desc); 816 else if (mt76_queue_is_npu_rx(q)) 817 size = sizeof(struct airoha_npu_rx_dma_desc); 818 else 819 size = sizeof(struct mt76_desc); 820 821 q->desc = dmam_alloc_coherent(dev->dma_dev, q->ndesc * size, 822 &q->desc_dma, GFP_KERNEL); 823 if (!q->desc) 824 return -ENOMEM; 825 826 mt76_dma_queue_magic_cnt_init(dev, q); 827 size = q->ndesc * sizeof(*q->entry); 828 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL); 829 if (!q->entry) 830 return -ENOMEM; 831 832 ret = mt76_create_page_pool(dev, q); 833 if (ret) 834 return ret; 835 836 mt76_npu_queue_setup(dev, q); 837 ret = mt76_wed_dma_setup(dev, q, false); 838 if (ret) 839 return ret; 840 841 if (mtk_wed_device_active(&dev->mmio.wed)) { 842 if ((mtk_wed_get_rx_capa(&dev->mmio.wed) && mt76_queue_is_wed_rro(q)) || 843 mt76_queue_is_wed_tx_free(q)) 844 return 0; 845 } 846 847 /* HW specific driver is supposed to reset brand-new EMI queues since 848 * it needs to set cpu index pointer. 849 */ 850 mt76_dma_queue_reset(dev, q, !mt76_queue_is_emi(q)); 851 852 return 0; 853 } 854 855 static void 856 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) 857 { 858 void *buf; 859 bool more; 860 861 if (!q->ndesc) 862 return; 863 864 if (mt76_queue_is_npu(q)) { 865 mt76_npu_queue_cleanup(dev, q); 866 return; 867 } 868 869 do { 870 spin_lock_bh(&q->lock); 871 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL); 872 spin_unlock_bh(&q->lock); 873 874 if (!buf) 875 break; 876 877 if (!mt76_queue_is_wed_rro(q)) 878 mt76_put_page_pool_buf(buf, false); 879 } while (1); 880 881 spin_lock_bh(&q->lock); 882 if (q->rx_head) { 883 dev_kfree_skb(q->rx_head); 884 q->rx_head = NULL; 885 } 886 887 spin_unlock_bh(&q->lock); 888 } 889 890 static void 891 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) 892 { 893 struct mt76_queue *q = &dev->q_rx[qid]; 894 895 if (!q->ndesc) 896 return; 897 898 if (!mt76_queue_is_wed_rro_ind(q) && 899 !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { 900 int i; 901 902 for (i = 0; i < q->ndesc; i++) 903 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); 904 } 905 906 mt76_dma_rx_cleanup(dev, q); 907 908 /* reset WED rx queues */ 909 mt76_wed_dma_setup(dev, q, true); 910 911 if (mt76_queue_is_wed_tx_free(q)) 912 return; 913 914 if (mtk_wed_device_active(&dev->mmio.wed) && 915 mt76_queue_is_wed_rro(q)) 916 return; 917 918 mt76_dma_sync_idx(dev, q); 919 if (mt76_queue_is_npu(q)) 920 mt76_npu_fill_rx_queue(dev, q); 921 else 922 mt76_dma_rx_fill(dev, q, false); 923 } 924 925 static void 926 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data, 927 int len, bool more, u32 info, bool allow_direct) 928 { 929 struct sk_buff *skb = q->rx_head; 930 struct skb_shared_info *shinfo = skb_shinfo(skb); 931 int nr_frags = shinfo->nr_frags; 932 933 if (nr_frags < ARRAY_SIZE(shinfo->frags)) { 934 struct page *page = virt_to_head_page(data); 935 int offset = data - page_address(page) + q->buf_offset; 936 937 skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size); 938 } else { 939 mt76_put_page_pool_buf(data, allow_direct); 940 } 941 942 if (more) 943 return; 944 945 q->rx_head = NULL; 946 if (nr_frags < ARRAY_SIZE(shinfo->frags)) 947 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info); 948 else 949 dev_kfree_skb(skb); 950 } 951 952 static int 953 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) 954 { 955 int len, data_len, done = 0, dma_idx; 956 struct sk_buff *skb; 957 unsigned char *data; 958 bool check_ddone = false; 959 bool allow_direct = !mt76_queue_is_wed_rx(q); 960 bool more; 961 962 if ((q->flags & MT_QFLAG_WED_RRO_EN) || 963 (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && 964 mt76_queue_is_wed_tx_free(q))) { 965 dma_idx = Q_READ(q, dma_idx); 966 check_ddone = true; 967 } 968 969 while (done < budget) { 970 bool drop = false; 971 u32 info; 972 973 if (check_ddone) { 974 if (q->tail == dma_idx) 975 dma_idx = Q_READ(q, dma_idx); 976 977 if (q->tail == dma_idx) 978 break; 979 } 980 981 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more, 982 &drop); 983 if (!data) 984 break; 985 986 if (PTR_ERR(data) == -EAGAIN) { 987 done++; 988 continue; 989 } 990 991 if (mt76_queue_is_wed_rro_ind(q) && dev->drv->rx_rro_ind_process) 992 dev->drv->rx_rro_ind_process(dev, data); 993 994 if (mt76_queue_is_wed_rro(q) && 995 !mt76_queue_is_wed_rro_rxdmad_c(q)) { 996 done++; 997 continue; 998 } 999 1000 if (drop) 1001 goto free_frag; 1002 1003 if (q->rx_head) 1004 data_len = q->buf_size; 1005 else 1006 data_len = SKB_WITH_OVERHEAD(q->buf_size); 1007 1008 if (data_len < len + q->buf_offset) { 1009 dev_kfree_skb(q->rx_head); 1010 q->rx_head = NULL; 1011 goto free_frag; 1012 } 1013 1014 if (q->rx_head) { 1015 mt76_add_fragment(dev, q, data, len, more, info, 1016 allow_direct); 1017 continue; 1018 } 1019 1020 if (!more && dev->drv->rx_check && 1021 !(dev->drv->rx_check(dev, data, len))) 1022 goto free_frag; 1023 1024 skb = napi_build_skb(data, q->buf_size); 1025 if (!skb) 1026 goto free_frag; 1027 1028 skb_reserve(skb, q->buf_offset); 1029 skb_mark_for_recycle(skb); 1030 1031 *(u32 *)skb->cb = info; 1032 1033 __skb_put(skb, len); 1034 done++; 1035 1036 if (more) { 1037 q->rx_head = skb; 1038 continue; 1039 } 1040 1041 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info); 1042 continue; 1043 1044 free_frag: 1045 mt76_put_page_pool_buf(data, allow_direct); 1046 } 1047 1048 mt76_dma_rx_fill(dev, q, true); 1049 return done; 1050 } 1051 1052 int mt76_dma_rx_poll(struct napi_struct *napi, int budget) 1053 { 1054 struct mt76_dev *dev; 1055 int qid, done = 0, cur; 1056 1057 dev = mt76_priv(napi->dev); 1058 qid = napi - dev->napi; 1059 1060 rcu_read_lock(); 1061 1062 do { 1063 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done); 1064 mt76_rx_poll_complete(dev, qid, napi); 1065 done += cur; 1066 } while (cur && done < budget); 1067 1068 rcu_read_unlock(); 1069 1070 if (done < budget && napi_complete(napi)) 1071 dev->drv->rx_poll_complete(dev, qid); 1072 1073 return done; 1074 } 1075 EXPORT_SYMBOL_GPL(mt76_dma_rx_poll); 1076 1077 static void 1078 mt76_dma_rx_queue_init(struct mt76_dev *dev, enum mt76_rxq_id qid, 1079 int (*poll)(struct napi_struct *napi, int budget)) 1080 { 1081 netif_napi_add(dev->napi_dev, &dev->napi[qid], poll); 1082 mt76_dma_rx_fill_buf(dev, &dev->q_rx[qid], false); 1083 napi_enable(&dev->napi[qid]); 1084 } 1085 1086 static int 1087 mt76_dma_init(struct mt76_dev *dev, 1088 int (*poll)(struct napi_struct *napi, int budget)) 1089 { 1090 struct mt76_dev **priv; 1091 int i; 1092 1093 dev->napi_dev = alloc_netdev_dummy(sizeof(struct mt76_dev *)); 1094 if (!dev->napi_dev) 1095 return -ENOMEM; 1096 1097 /* napi_dev private data points to mt76_dev parent, so, mt76_dev 1098 * can be retrieved given napi_dev 1099 */ 1100 priv = netdev_priv(dev->napi_dev); 1101 *priv = dev; 1102 1103 dev->tx_napi_dev = alloc_netdev_dummy(sizeof(struct mt76_dev *)); 1104 if (!dev->tx_napi_dev) { 1105 free_netdev(dev->napi_dev); 1106 return -ENOMEM; 1107 } 1108 priv = netdev_priv(dev->tx_napi_dev); 1109 *priv = dev; 1110 1111 snprintf(dev->napi_dev->name, sizeof(dev->napi_dev->name), "%s", 1112 wiphy_name(dev->hw->wiphy)); 1113 dev->napi_dev->threaded = 1; 1114 init_completion(&dev->mmio.wed_reset); 1115 init_completion(&dev->mmio.wed_reset_complete); 1116 1117 mt76_for_each_q_rx(dev, i) { 1118 if (mt76_queue_is_wed_rro(&dev->q_rx[i])) 1119 continue; 1120 1121 mt76_dma_rx_queue_init(dev, i, poll); 1122 } 1123 1124 return 0; 1125 } 1126 1127 static const struct mt76_queue_ops mt76_dma_ops = { 1128 .init = mt76_dma_init, 1129 .alloc = mt76_dma_alloc_queue, 1130 .reset_q = mt76_dma_queue_reset, 1131 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw, 1132 .tx_queue_skb = mt76_dma_tx_queue_skb, 1133 .tx_cleanup = mt76_dma_tx_cleanup, 1134 .rx_queue_init = mt76_dma_rx_queue_init, 1135 .rx_cleanup = mt76_dma_rx_cleanup, 1136 .rx_reset = mt76_dma_rx_reset, 1137 .kick = mt76_dma_kick_queue, 1138 }; 1139 1140 void mt76_dma_attach(struct mt76_dev *dev) 1141 { 1142 dev->queue_ops = &mt76_dma_ops; 1143 } 1144 EXPORT_SYMBOL_GPL(mt76_dma_attach); 1145 1146 void mt76_dma_cleanup(struct mt76_dev *dev) 1147 { 1148 int i; 1149 1150 mt76_worker_disable(&dev->tx_worker); 1151 napi_disable(&dev->tx_napi); 1152 netif_napi_del(&dev->tx_napi); 1153 1154 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 1155 struct mt76_phy *phy = dev->phys[i]; 1156 int j; 1157 1158 if (!phy) 1159 continue; 1160 1161 for (j = 0; j < ARRAY_SIZE(phy->q_tx); j++) 1162 mt76_dma_tx_cleanup(dev, phy->q_tx[j], true); 1163 } 1164 1165 for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++) 1166 mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true); 1167 1168 mt76_for_each_q_rx(dev, i) { 1169 struct mt76_queue *q = &dev->q_rx[i]; 1170 1171 if (mtk_wed_device_active(&dev->mmio.wed) && 1172 mt76_queue_is_wed_rro(q)) 1173 continue; 1174 1175 netif_napi_del(&dev->napi[i]); 1176 mt76_dma_rx_cleanup(dev, q); 1177 1178 page_pool_destroy(q->page_pool); 1179 } 1180 1181 if (mtk_wed_device_active(&dev->mmio.wed)) 1182 mtk_wed_device_detach(&dev->mmio.wed); 1183 1184 if (mtk_wed_device_active(&dev->mmio.wed_hif2)) 1185 mtk_wed_device_detach(&dev->mmio.wed_hif2); 1186 1187 mt76_free_pending_txwi(dev); 1188 mt76_free_pending_rxwi(dev); 1189 free_netdev(dev->napi_dev); 1190 free_netdev(dev->tx_napi_dev); 1191 } 1192 EXPORT_SYMBOL_GPL(mt76_dma_cleanup); 1193