1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include <linux/dma-mapping.h> 7 #include "mt76.h" 8 #include "dma.h" 9 #include "mt76_connac.h" 10 11 static struct mt76_txwi_cache * 12 mt76_alloc_txwi(struct mt76_dev *dev) 13 { 14 struct mt76_txwi_cache *t; 15 dma_addr_t addr; 16 u8 *txwi; 17 int size; 18 19 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t)); 20 txwi = kzalloc(size, GFP_ATOMIC); 21 if (!txwi) 22 return NULL; 23 24 addr = dma_map_single(dev->dma_dev, txwi, dev->drv->txwi_size, 25 DMA_TO_DEVICE); 26 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) { 27 kfree(txwi); 28 return NULL; 29 } 30 31 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size); 32 t->dma_addr = addr; 33 34 return t; 35 } 36 37 static struct mt76_txwi_cache * 38 mt76_alloc_rxwi(struct mt76_dev *dev) 39 { 40 struct mt76_txwi_cache *t; 41 42 t = kzalloc(L1_CACHE_ALIGN(sizeof(*t)), GFP_ATOMIC); 43 if (!t) 44 return NULL; 45 46 t->ptr = NULL; 47 return t; 48 } 49 50 static struct mt76_txwi_cache * 51 __mt76_get_txwi(struct mt76_dev *dev) 52 { 53 struct mt76_txwi_cache *t = NULL; 54 55 spin_lock(&dev->lock); 56 if (!list_empty(&dev->txwi_cache)) { 57 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache, 58 list); 59 list_del(&t->list); 60 } 61 spin_unlock(&dev->lock); 62 63 return t; 64 } 65 66 static struct mt76_txwi_cache * 67 __mt76_get_rxwi(struct mt76_dev *dev) 68 { 69 struct mt76_txwi_cache *t = NULL; 70 71 spin_lock_bh(&dev->wed_lock); 72 if (!list_empty(&dev->rxwi_cache)) { 73 t = list_first_entry(&dev->rxwi_cache, struct mt76_txwi_cache, 74 list); 75 list_del(&t->list); 76 } 77 spin_unlock_bh(&dev->wed_lock); 78 79 return t; 80 } 81 82 static struct mt76_txwi_cache * 83 mt76_get_txwi(struct mt76_dev *dev) 84 { 85 struct mt76_txwi_cache *t = __mt76_get_txwi(dev); 86 87 if (t) 88 return t; 89 90 return mt76_alloc_txwi(dev); 91 } 92 93 struct mt76_txwi_cache * 94 mt76_get_rxwi(struct mt76_dev *dev) 95 { 96 struct mt76_txwi_cache *t = __mt76_get_rxwi(dev); 97 98 if (t) 99 return t; 100 101 return mt76_alloc_rxwi(dev); 102 } 103 EXPORT_SYMBOL_GPL(mt76_get_rxwi); 104 105 void 106 mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) 107 { 108 if (!t) 109 return; 110 111 spin_lock(&dev->lock); 112 list_add(&t->list, &dev->txwi_cache); 113 spin_unlock(&dev->lock); 114 } 115 EXPORT_SYMBOL_GPL(mt76_put_txwi); 116 117 void 118 mt76_put_rxwi(struct mt76_dev *dev, struct mt76_txwi_cache *t) 119 { 120 if (!t) 121 return; 122 123 spin_lock_bh(&dev->wed_lock); 124 list_add(&t->list, &dev->rxwi_cache); 125 spin_unlock_bh(&dev->wed_lock); 126 } 127 EXPORT_SYMBOL_GPL(mt76_put_rxwi); 128 129 static void 130 mt76_free_pending_txwi(struct mt76_dev *dev) 131 { 132 struct mt76_txwi_cache *t; 133 134 local_bh_disable(); 135 while ((t = __mt76_get_txwi(dev)) != NULL) { 136 dma_unmap_single(dev->dma_dev, t->dma_addr, dev->drv->txwi_size, 137 DMA_TO_DEVICE); 138 kfree(mt76_get_txwi_ptr(dev, t)); 139 } 140 local_bh_enable(); 141 } 142 143 void 144 mt76_free_pending_rxwi(struct mt76_dev *dev) 145 { 146 struct mt76_txwi_cache *t; 147 148 local_bh_disable(); 149 while ((t = __mt76_get_rxwi(dev)) != NULL) { 150 if (t->ptr) 151 mt76_put_page_pool_buf(t->ptr, false); 152 kfree(t); 153 } 154 local_bh_enable(); 155 } 156 EXPORT_SYMBOL_GPL(mt76_free_pending_rxwi); 157 158 static void 159 mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q) 160 { 161 if (!mt76_queue_is_wed_rro(q)) 162 return; 163 164 q->magic_cnt = 0; 165 if (mt76_queue_is_wed_rro_ind(q)) { 166 struct mt76_wed_rro_desc *rro_desc; 167 u32 data1 = FIELD_PREP(RRO_IND_DATA1_MAGIC_CNT_MASK, 168 MT_DMA_WED_IND_CMD_CNT - 1); 169 int i; 170 171 rro_desc = (struct mt76_wed_rro_desc *)q->desc; 172 for (i = 0; i < q->ndesc; i++) { 173 struct mt76_wed_rro_ind *cmd; 174 175 cmd = (struct mt76_wed_rro_ind *)&rro_desc[i]; 176 cmd->data1 = cpu_to_le32(data1); 177 } 178 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { 179 struct mt76_rro_rxdmad_c *dmad = (void *)q->desc; 180 u32 data3 = FIELD_PREP(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK, 181 MT_DMA_MAGIC_CNT - 1); 182 int i; 183 184 for (i = 0; i < q->ndesc; i++) 185 dmad[i].data3 = cpu_to_le32(data3); 186 } 187 } 188 189 /* A hung bus (e.g. after a PCIe AER error) reads 0xffffffff from every 190 * register, so clamp an out-of-range index to the fallback to keep it from 191 * corrupting q->head/q->tail. 192 */ 193 static int 194 mt76_dma_read_dma_idx(struct mt76_queue *q, int fallback) 195 { 196 u32 idx = Q_READ(q, dma_idx); 197 198 return idx < q->ndesc ? idx : fallback; 199 } 200 201 static void 202 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) 203 { 204 if ((q->flags & MT_QFLAG_WED_RRO_EN) && 205 (!is_mt7992(dev) || !mt76_npu_device_active(dev))) 206 Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc); 207 else 208 Q_WRITE(q, ring_size, q->ndesc); 209 210 if (mt76_queue_is_npu_tx(q)) { 211 writel(q->ndesc, &q->regs->ring_size); 212 writel(q->desc_dma, &q->regs->desc_base); 213 } 214 215 Q_WRITE(q, desc_base, q->desc_dma); 216 217 q->head = mt76_dma_read_dma_idx(q, 0); 218 q->tail = q->head; 219 } 220 221 void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, 222 bool reset_idx) 223 { 224 if (!q || !q->ndesc) 225 return; 226 227 if (!mt76_queue_is_wed_rro_ind(q) && 228 !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { 229 int i; 230 231 /* clear descriptors */ 232 for (i = 0; i < q->ndesc; i++) 233 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); 234 } 235 236 mt76_dma_queue_magic_cnt_init(dev, q); 237 if (reset_idx) { 238 if (mt76_queue_is_emi(q)) 239 *q->emi_cpu_idx = 0; 240 else 241 Q_WRITE(q, cpu_idx, 0); 242 Q_WRITE(q, dma_idx, 0); 243 } 244 mt76_dma_sync_idx(dev, q); 245 } 246 247 static int 248 mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q, 249 struct mt76_queue_buf *buf, void *data) 250 { 251 struct mt76_queue_entry *entry = &q->entry[q->head]; 252 struct mt76_txwi_cache *txwi = NULL; 253 u32 buf1 = 0, ctrl, info = 0; 254 struct mt76_desc *desc; 255 int idx = q->head; 256 int rx_token; 257 258 if (mt76_queue_is_wed_rro_ind(q)) { 259 struct mt76_wed_rro_desc *rro_desc; 260 261 rro_desc = (struct mt76_wed_rro_desc *)q->desc; 262 data = &rro_desc[q->head]; 263 goto done; 264 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { 265 data = &q->desc[q->head]; 266 goto done; 267 } 268 269 desc = &q->desc[q->head]; 270 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); 271 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 272 buf1 = FIELD_PREP(MT_DMA_CTL_SDP0_H, buf->addr >> 32); 273 #endif 274 275 if (mt76_queue_is_wed_rx(q) || mt76_queue_is_wed_rro_data(q)) { 276 txwi = mt76_get_rxwi(dev); 277 if (!txwi) 278 return -ENOMEM; 279 280 rx_token = mt76_rx_token_consume(dev, data, txwi, buf->addr); 281 if (rx_token < 0) { 282 mt76_put_rxwi(dev, txwi); 283 return -ENOMEM; 284 } 285 286 buf1 |= FIELD_PREP(MT_DMA_CTL_TOKEN, rx_token); 287 ctrl |= MT_DMA_CTL_TO_HOST; 288 289 txwi->qid = q - dev->q_rx; 290 } 291 292 if (mt76_queue_is_wed_rro_msdu_pg(q) && 293 dev->drv->rx_rro_add_msdu_page) { 294 if (dev->drv->rx_rro_add_msdu_page(dev, q, buf->addr, data)) 295 return -ENOMEM; 296 } 297 298 if (q->flags & MT_QFLAG_WED_RRO_EN) { 299 info |= FIELD_PREP(MT_DMA_MAGIC_MASK, q->magic_cnt); 300 if ((q->head + 1) == q->ndesc) 301 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT; 302 } 303 304 WRITE_ONCE(desc->buf0, cpu_to_le32(buf->addr)); 305 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); 306 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); 307 WRITE_ONCE(desc->info, cpu_to_le32(info)); 308 309 done: 310 entry->dma_addr[0] = buf->addr; 311 entry->dma_len[0] = buf->len; 312 entry->txwi = txwi; 313 entry->buf = data; 314 entry->wcid = 0xffff; 315 entry->skip_buf1 = true; 316 q->head = (q->head + 1) % q->ndesc; 317 q->queued++; 318 319 return idx; 320 } 321 322 static int 323 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q, 324 struct mt76_queue_buf *buf, int nbufs, u32 info, 325 struct sk_buff *skb, void *txwi) 326 { 327 struct mt76_queue_entry *entry; 328 struct mt76_desc *desc; 329 int i, idx = -1; 330 u32 ctrl, next; 331 332 if (txwi) { 333 q->entry[q->head].txwi = DMA_DUMMY_DATA; 334 q->entry[q->head].skip_buf0 = true; 335 } 336 337 for (i = 0; i < nbufs; i += 2, buf += 2) { 338 u32 buf0 = buf[0].addr, buf1 = 0; 339 340 idx = q->head; 341 next = (q->head + 1) % q->ndesc; 342 343 desc = &q->desc[idx]; 344 entry = &q->entry[idx]; 345 346 if (buf[0].skip_unmap) 347 entry->skip_buf0 = true; 348 entry->skip_buf1 = i == nbufs - 1; 349 350 entry->dma_addr[0] = buf[0].addr; 351 entry->dma_len[0] = buf[0].len; 352 353 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len); 354 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 355 info |= FIELD_PREP(MT_DMA_CTL_SDP0_H, buf[0].addr >> 32); 356 #endif 357 if (i < nbufs - 1) { 358 entry->dma_addr[1] = buf[1].addr; 359 entry->dma_len[1] = buf[1].len; 360 buf1 = buf[1].addr; 361 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len); 362 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 363 info |= FIELD_PREP(MT_DMA_CTL_SDP1_H, 364 buf[1].addr >> 32); 365 #endif 366 if (buf[1].skip_unmap) 367 entry->skip_buf1 = true; 368 } 369 370 if (i == nbufs - 1) 371 ctrl |= MT_DMA_CTL_LAST_SEC0; 372 else if (i == nbufs - 2) 373 ctrl |= MT_DMA_CTL_LAST_SEC1; 374 375 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0)); 376 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1)); 377 WRITE_ONCE(desc->info, cpu_to_le32(info)); 378 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl)); 379 380 q->head = next; 381 q->queued++; 382 } 383 384 q->entry[idx].txwi = txwi; 385 q->entry[idx].skb = skb; 386 q->entry[idx].wcid = 0xffff; 387 388 return idx; 389 } 390 391 static void 392 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx, 393 struct mt76_queue_entry *prev_e) 394 { 395 struct mt76_queue_entry *e = &q->entry[idx]; 396 397 if (!e->skip_buf0) 398 dma_unmap_single(dev->dma_dev, e->dma_addr[0], e->dma_len[0], 399 DMA_TO_DEVICE); 400 401 if (!e->skip_buf1) 402 dma_unmap_single(dev->dma_dev, e->dma_addr[1], e->dma_len[1], 403 DMA_TO_DEVICE); 404 405 if (e->txwi == DMA_DUMMY_DATA) 406 e->txwi = NULL; 407 408 *prev_e = *e; 409 memset(e, 0, sizeof(*e)); 410 } 411 412 static void 413 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q) 414 { 415 wmb(); 416 if (mt76_queue_is_emi(q)) 417 *q->emi_cpu_idx = cpu_to_le16(q->head); 418 else 419 Q_WRITE(q, cpu_idx, q->head); 420 } 421 422 static void 423 mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) 424 { 425 struct mt76_queue_entry entry; 426 int last; 427 428 if (!q || !q->ndesc) 429 return; 430 431 spin_lock_bh(&q->cleanup_lock); 432 if (flush) 433 last = -1; 434 else 435 last = mt76_dma_read_dma_idx(q, -1); 436 437 while (q->queued > 0 && q->tail != last) { 438 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); 439 mt76_npu_txdesc_cleanup(q, q->tail); 440 mt76_queue_tx_complete(dev, q, &entry); 441 442 if (entry.txwi) { 443 if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE)) 444 mt76_put_txwi(dev, entry.txwi); 445 } 446 447 if (!flush && q->tail == last) 448 last = mt76_dma_read_dma_idx(q, -1); 449 } 450 spin_unlock_bh(&q->cleanup_lock); 451 452 if (flush) { 453 spin_lock_bh(&q->lock); 454 mt76_dma_sync_idx(dev, q); 455 mt76_dma_kick_queue(dev, q); 456 spin_unlock_bh(&q->lock); 457 } 458 459 if (!q->queued) 460 wake_up(&dev->tx_wait); 461 } 462 463 static void * 464 mt76_dma_get_rxdmad_c_buf(struct mt76_dev *dev, struct mt76_queue *q, 465 int idx, int *len, bool *more) 466 { 467 struct mt76_queue_entry *e = &q->entry[idx]; 468 struct mt76_rro_rxdmad_c *dmad = e->buf; 469 u32 data1 = le32_to_cpu(dmad->data1); 470 u32 data2 = le32_to_cpu(dmad->data2); 471 struct mt76_txwi_cache *t; 472 u16 rx_token_id; 473 u8 ind_reason; 474 void *buf; 475 476 rx_token_id = FIELD_GET(RRO_RXDMAD_DATA2_RX_TOKEN_ID_MASK, data2); 477 t = mt76_rx_token_release(dev, rx_token_id); 478 if (!t) 479 return ERR_PTR(-EAGAIN); 480 481 q = &dev->q_rx[t->qid]; 482 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, 483 SKB_WITH_OVERHEAD(q->buf_size), 484 page_pool_get_dma_dir(q->page_pool)); 485 486 if (len) 487 *len = FIELD_GET(RRO_RXDMAD_DATA1_SDL0_MASK, data1); 488 if (more) 489 *more = !FIELD_GET(RRO_RXDMAD_DATA1_LS_MASK, data1); 490 491 buf = t->ptr; 492 ind_reason = FIELD_GET(RRO_RXDMAD_DATA2_IND_REASON_MASK, data2); 493 if (ind_reason == MT_DMA_WED_IND_REASON_REPEAT || 494 ind_reason == MT_DMA_WED_IND_REASON_OLDPKT) { 495 mt76_put_page_pool_buf(buf, false); 496 buf = ERR_PTR(-EAGAIN); 497 } 498 t->ptr = NULL; 499 t->dma_addr = 0; 500 501 mt76_put_rxwi(dev, t); 502 503 return buf; 504 } 505 506 static void * 507 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, 508 int *len, u32 *info, bool *more, bool *drop, bool flush) 509 { 510 struct mt76_queue_entry *e = &q->entry[idx]; 511 struct mt76_desc *desc = &q->desc[idx]; 512 u32 ctrl, desc_info, buf1; 513 void *buf = e->buf; 514 515 if (mt76_queue_is_wed_rro_rxdmad_c(q) && !flush) 516 buf = mt76_dma_get_rxdmad_c_buf(dev, q, idx, len, more); 517 518 if (mt76_queue_is_wed_rro(q)) 519 goto done; 520 521 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); 522 if (len) { 523 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl); 524 *more = !(ctrl & MT_DMA_CTL_LAST_SEC0); 525 } 526 527 desc_info = le32_to_cpu(desc->info); 528 if (info) 529 *info = desc_info; 530 531 buf1 = le32_to_cpu(desc->buf1); 532 mt76_dma_should_drop_buf(drop, ctrl, buf1, desc_info); 533 534 if (mt76_queue_is_wed_rx(q)) { 535 u32 token = FIELD_GET(MT_DMA_CTL_TOKEN, buf1); 536 struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token); 537 538 if (!t) 539 return NULL; 540 541 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, 542 SKB_WITH_OVERHEAD(q->buf_size), 543 page_pool_get_dma_dir(q->page_pool)); 544 545 buf = t->ptr; 546 t->dma_addr = 0; 547 t->ptr = NULL; 548 549 mt76_put_rxwi(dev, t); 550 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 551 /* the WO MCU owns the RX path only on WED v2, on newer 552 * versions this buf1 bit carries no drop information 553 */ 554 if (drop && dev->mmio.wed.version == 2) 555 *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP); 556 #endif 557 } else { 558 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], 559 SKB_WITH_OVERHEAD(q->buf_size), 560 page_pool_get_dma_dir(q->page_pool)); 561 } 562 563 done: 564 e->buf = NULL; 565 return buf; 566 } 567 568 static void * 569 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, 570 int *len, u32 *info, bool *more, bool *drop) 571 { 572 int idx = q->tail; 573 574 *more = false; 575 if (!q->queued) 576 return NULL; 577 578 if (mt76_queue_is_wed_rro_data(q) || mt76_queue_is_wed_rro_msdu_pg(q)) 579 goto done; 580 581 if (mt76_queue_is_wed_rro_ind(q)) { 582 struct mt76_wed_rro_ind *cmd; 583 u8 magic_cnt; 584 585 if (flush) 586 goto done; 587 588 cmd = q->entry[idx].buf; 589 magic_cnt = FIELD_GET(RRO_IND_DATA1_MAGIC_CNT_MASK, 590 le32_to_cpu(cmd->data1)); 591 if (magic_cnt != q->magic_cnt) 592 return NULL; 593 594 if (q->tail == q->ndesc - 1) 595 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_WED_IND_CMD_CNT; 596 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) { 597 struct mt76_rro_rxdmad_c *dmad; 598 u16 magic_cnt; 599 600 if (flush) 601 goto done; 602 603 dmad = q->entry[idx].buf; 604 magic_cnt = FIELD_GET(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK, 605 le32_to_cpu(dmad->data3)); 606 if (magic_cnt != q->magic_cnt) 607 return NULL; 608 609 if (q->tail == q->ndesc - 1) 610 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT; 611 } else { 612 if (flush) 613 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); 614 else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) 615 return NULL; 616 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 617 /* on WED v3 the M_DONE bit signals that WED is done reading 618 * the txfree descriptor; WED v2 does not set it 619 */ 620 else if (dev->mmio.wed.version > 2 && 621 mt76_queue_is_wed_tx_free(q) && 622 !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_M_DONE))) 623 return NULL; 624 #endif 625 } 626 done: 627 q->tail = (q->tail + 1) % q->ndesc; 628 q->queued--; 629 630 return mt76_dma_get_buf(dev, q, idx, len, info, more, drop, flush); 631 } 632 633 static int 634 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q, 635 struct sk_buff *skb, u32 tx_info) 636 { 637 struct mt76_queue_buf buf = {}; 638 dma_addr_t addr; 639 640 if (test_bit(MT76_MCU_RESET, &dev->phy.state)) 641 goto error; 642 643 if (q->queued + 1 >= q->ndesc - 1) 644 goto error; 645 646 addr = dma_map_single(dev->dma_dev, skb->data, skb->len, 647 DMA_TO_DEVICE); 648 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) 649 goto error; 650 651 buf.addr = addr; 652 buf.len = skb->len; 653 654 spin_lock_bh(&q->lock); 655 if (mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL) >= 0) 656 mt76_dma_kick_queue(dev, q); 657 spin_unlock_bh(&q->lock); 658 659 return 0; 660 661 error: 662 dev_kfree_skb(skb); 663 return -ENOMEM; 664 } 665 666 static int 667 mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, 668 enum mt76_txq_id qid, struct sk_buff *skb, 669 struct mt76_wcid *wcid, struct ieee80211_sta *sta) 670 { 671 struct ieee80211_tx_status status = { 672 .sta = sta, 673 }; 674 struct mt76_tx_info tx_info = { 675 .skb = skb, 676 }; 677 struct mt76_dev *dev = phy->dev; 678 struct ieee80211_hw *hw; 679 int len, n = 0, ret = -ENOMEM; 680 struct mt76_txwi_cache *t; 681 struct sk_buff *iter; 682 dma_addr_t addr; 683 u8 *txwi; 684 685 if (test_bit(MT76_RESET, &phy->state)) 686 goto free_skb; 687 688 /* TODO: Take into account unlinear skbs */ 689 if (mt76_npu_device_active(dev) && skb_linearize(skb)) 690 goto free_skb; 691 692 t = mt76_get_txwi(dev); 693 if (!t) 694 goto free_skb; 695 696 t->phy_idx = phy->band_idx; 697 t->qid = qid; 698 txwi = mt76_get_txwi_ptr(dev, t); 699 700 skb->prev = skb->next = NULL; 701 if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS) 702 mt76_insert_hdr_pad(skb); 703 704 len = skb_headlen(skb); 705 addr = dma_map_single(dev->dma_dev, skb->data, len, DMA_TO_DEVICE); 706 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) 707 goto free; 708 709 tx_info.buf[n].addr = t->dma_addr; 710 tx_info.buf[n++].len = dev->drv->txwi_size; 711 tx_info.buf[n].addr = addr; 712 tx_info.buf[n++].len = len; 713 714 skb_walk_frags(skb, iter) { 715 if (n == ARRAY_SIZE(tx_info.buf)) 716 goto unmap; 717 718 addr = dma_map_single(dev->dma_dev, iter->data, iter->len, 719 DMA_TO_DEVICE); 720 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) 721 goto unmap; 722 723 tx_info.buf[n].addr = addr; 724 tx_info.buf[n++].len = iter->len; 725 } 726 tx_info.nbuf = n; 727 728 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) { 729 ret = -ENOMEM; 730 goto unmap; 731 } 732 733 dma_sync_single_for_cpu(dev->dma_dev, t->dma_addr, dev->drv->txwi_size, 734 DMA_TO_DEVICE); 735 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info); 736 dma_sync_single_for_device(dev->dma_dev, t->dma_addr, dev->drv->txwi_size, 737 DMA_TO_DEVICE); 738 if (ret < 0) 739 goto unmap; 740 741 if (mt76_npu_device_active(dev)) 742 return mt76_npu_dma_add_buf(phy, q, skb, &tx_info.buf[1], txwi); 743 744 return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf, 745 tx_info.info, tx_info.skb, t); 746 747 unmap: 748 for (n--; n > 0; n--) 749 dma_unmap_single(dev->dma_dev, tx_info.buf[n].addr, 750 tx_info.buf[n].len, DMA_TO_DEVICE); 751 752 free: 753 #ifdef CONFIG_NL80211_TESTMODE 754 /* fix tx_done accounting on queue overflow */ 755 if (mt76_is_testmode_skb(dev, skb, &hw)) { 756 struct mt76_phy *phy = hw->priv; 757 758 if (tx_info.skb == phy->test.tx_skb) 759 phy->test.tx_done--; 760 } 761 #endif 762 763 mt76_put_txwi(dev, t); 764 765 free_skb: 766 status.skb = tx_info.skb; 767 hw = mt76_tx_status_get_hw(dev, tx_info.skb); 768 spin_lock_bh(&dev->rx_lock); 769 ieee80211_tx_status_ext(hw, &status); 770 spin_unlock_bh(&dev->rx_lock); 771 772 return ret; 773 } 774 775 static int 776 mt76_dma_rx_fill_buf(struct mt76_dev *dev, struct mt76_queue *q, 777 bool allow_direct) 778 { 779 int len = SKB_WITH_OVERHEAD(q->buf_size); 780 int frames = 0; 781 782 if (!q->ndesc) 783 return 0; 784 785 while (q->queued < q->ndesc - 1) { 786 struct mt76_queue_buf qbuf = {}; 787 void *buf = NULL; 788 int offset; 789 790 if (mt76_queue_is_wed_rro_ind(q) || 791 mt76_queue_is_wed_rro_rxdmad_c(q)) 792 goto done; 793 794 buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); 795 if (!buf) 796 break; 797 798 qbuf.addr = page_pool_get_dma_addr(virt_to_head_page(buf)) + 799 offset + q->buf_offset; 800 done: 801 qbuf.len = len - q->buf_offset; 802 qbuf.skip_unmap = false; 803 if (mt76_dma_add_rx_buf(dev, q, &qbuf, buf) < 0) { 804 mt76_put_page_pool_buf(buf, allow_direct); 805 break; 806 } 807 frames++; 808 } 809 810 if (frames || mt76_queue_is_wed_rx(q)) 811 mt76_dma_kick_queue(dev, q); 812 813 return frames; 814 } 815 816 int mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, 817 bool allow_direct) 818 { 819 int frames; 820 821 spin_lock_bh(&q->lock); 822 frames = mt76_dma_rx_fill_buf(dev, q, allow_direct); 823 spin_unlock_bh(&q->lock); 824 825 return frames; 826 } 827 828 static int 829 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, 830 int idx, int n_desc, int bufsize, 831 u32 ring_base) 832 { 833 int ret, size; 834 835 spin_lock_init(&q->lock); 836 spin_lock_init(&q->cleanup_lock); 837 838 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE; 839 q->ndesc = n_desc; 840 q->buf_size = bufsize; 841 q->hw_idx = idx; 842 q->dev = dev; 843 844 if (mt76_queue_is_wed_rro_ind(q)) 845 size = sizeof(struct mt76_wed_rro_desc); 846 else if (mt76_queue_is_npu_tx(q)) 847 size = sizeof(struct airoha_npu_tx_dma_desc); 848 else if (mt76_queue_is_npu_rx(q)) 849 size = sizeof(struct airoha_npu_rx_dma_desc); 850 else 851 size = sizeof(struct mt76_desc); 852 853 q->desc = dmam_alloc_coherent(dev->dma_dev, q->ndesc * size, 854 &q->desc_dma, GFP_KERNEL); 855 if (!q->desc) 856 return -ENOMEM; 857 858 mt76_dma_queue_magic_cnt_init(dev, q); 859 size = q->ndesc * sizeof(*q->entry); 860 q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL); 861 if (!q->entry) 862 return -ENOMEM; 863 864 ret = mt76_create_page_pool(dev, q); 865 if (ret) 866 return ret; 867 868 mt76_npu_queue_setup(dev, q); 869 ret = mt76_wed_dma_setup(dev, q, false); 870 if (ret) 871 return ret; 872 873 if (mtk_wed_device_active(&dev->mmio.wed)) { 874 if ((mtk_wed_get_rx_capa(&dev->mmio.wed) && mt76_queue_is_wed_rro(q)) || 875 mt76_queue_is_wed_tx_free(q)) 876 return 0; 877 } 878 879 /* HW specific driver is supposed to reset brand-new EMI queues since 880 * it needs to set cpu index pointer. 881 */ 882 mt76_dma_queue_reset(dev, q, !mt76_queue_is_emi(q)); 883 884 return 0; 885 } 886 887 static void 888 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) 889 { 890 void *buf; 891 bool more; 892 893 if (!q->ndesc) 894 return; 895 896 if (mt76_queue_is_npu(q)) { 897 mt76_npu_queue_cleanup(dev, q); 898 return; 899 } 900 901 do { 902 spin_lock_bh(&q->lock); 903 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL); 904 spin_unlock_bh(&q->lock); 905 906 if (!buf) 907 break; 908 909 if (mtk_wed_device_active(&dev->mmio.wed) && 910 mt76_queue_is_wed_rro(q)) 911 continue; 912 913 if (mt76_npu_device_active(dev) && 914 mt76_queue_is_wed_rro(q)) 915 continue; 916 917 if (!mt76_queue_is_wed_rro_rxdmad_c(q) && 918 !mt76_queue_is_wed_rro_ind(q)) 919 mt76_put_page_pool_buf(buf, false); 920 } while (1); 921 922 spin_lock_bh(&q->lock); 923 if (q->rx_head) { 924 dev_kfree_skb(q->rx_head); 925 q->rx_head = NULL; 926 } 927 928 spin_unlock_bh(&q->lock); 929 } 930 931 static void 932 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) 933 { 934 struct mt76_queue *q = &dev->q_rx[qid]; 935 936 if (!q->ndesc) 937 return; 938 939 if (!mt76_queue_is_wed_rro_ind(q) && 940 !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { 941 int i; 942 943 for (i = 0; i < q->ndesc; i++) 944 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE); 945 } 946 947 mt76_dma_rx_cleanup(dev, q); 948 949 /* reset WED rx queues */ 950 mt76_wed_dma_setup(dev, q, true); 951 952 if (mt76_queue_is_wed_tx_free(q)) 953 return; 954 955 if (mtk_wed_device_active(&dev->mmio.wed) && 956 mt76_queue_is_wed_rro(q)) 957 return; 958 959 if (mt76_npu_device_active(dev) && 960 mt76_queue_is_wed_rro(q)) 961 return; 962 963 if (mt76_queue_is_npu_txfree(q)) 964 return; 965 966 mt76_dma_sync_idx(dev, q); 967 if (mt76_queue_is_npu(q)) 968 mt76_npu_fill_rx_queue(dev, q); 969 else 970 mt76_dma_rx_fill(dev, q, false); 971 } 972 973 static void 974 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data, 975 int len, bool more, u32 info, bool allow_direct) 976 { 977 struct sk_buff *skb = q->rx_head; 978 struct skb_shared_info *shinfo = skb_shinfo(skb); 979 int nr_frags = shinfo->nr_frags; 980 981 if (nr_frags < ARRAY_SIZE(shinfo->frags)) { 982 struct page *page = virt_to_head_page(data); 983 int offset = data - page_address(page) + q->buf_offset; 984 985 skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size); 986 } else { 987 mt76_put_page_pool_buf(data, allow_direct); 988 } 989 990 if (more) 991 return; 992 993 q->rx_head = NULL; 994 if (nr_frags < ARRAY_SIZE(shinfo->frags)) 995 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info); 996 else 997 dev_kfree_skb(skb); 998 } 999 1000 static int 1001 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) 1002 { 1003 int len, data_len, done = 0, dma_idx; 1004 struct sk_buff *skb; 1005 unsigned char *data; 1006 bool check_ddone = false; 1007 bool allow_direct = !mt76_queue_is_wed_rx(q) && 1008 !mt76_queue_is_wed_rro_rxdmad_c(q); 1009 bool more; 1010 1011 if ((q->flags & MT_QFLAG_WED_RRO_EN) || 1012 (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && 1013 mt76_queue_is_wed_tx_free(q))) { 1014 dma_idx = mt76_dma_read_dma_idx(q, q->tail); 1015 check_ddone = true; 1016 } 1017 1018 while (done < budget) { 1019 bool drop = false; 1020 u32 info = 0; 1021 1022 if (check_ddone) { 1023 if (q->tail == dma_idx) 1024 dma_idx = mt76_dma_read_dma_idx(q, q->tail); 1025 1026 if (q->tail == dma_idx) 1027 break; 1028 } 1029 1030 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more, 1031 &drop); 1032 if (!data) 1033 break; 1034 1035 if (PTR_ERR(data) == -EAGAIN) { 1036 done++; 1037 continue; 1038 } 1039 1040 if (mt76_queue_is_wed_rro_ind(q) && dev->drv->rx_rro_ind_process) 1041 dev->drv->rx_rro_ind_process(dev, data); 1042 1043 if (mt76_queue_is_wed_rro(q) && 1044 !mt76_queue_is_wed_rro_rxdmad_c(q)) { 1045 done++; 1046 continue; 1047 } 1048 1049 if (drop) 1050 goto free_frag; 1051 1052 if (q->rx_head) 1053 data_len = q->buf_size; 1054 else 1055 data_len = SKB_WITH_OVERHEAD(q->buf_size); 1056 1057 if (data_len < len + q->buf_offset) { 1058 dev_kfree_skb(q->rx_head); 1059 q->rx_head = NULL; 1060 goto free_frag; 1061 } 1062 1063 if (q->rx_head) { 1064 mt76_add_fragment(dev, q, data, len, more, info, 1065 allow_direct); 1066 continue; 1067 } 1068 1069 if (!more && dev->drv->rx_check && 1070 !(dev->drv->rx_check(dev, data, len))) 1071 goto free_frag; 1072 1073 skb = napi_build_skb(data, q->buf_size); 1074 if (!skb) 1075 goto free_frag; 1076 1077 skb_reserve(skb, q->buf_offset); 1078 skb_mark_for_recycle(skb); 1079 1080 *(u32 *)skb->cb = info; 1081 1082 __skb_put(skb, len); 1083 done++; 1084 1085 if (more) { 1086 q->rx_head = skb; 1087 continue; 1088 } 1089 1090 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info); 1091 continue; 1092 1093 free_frag: 1094 mt76_put_page_pool_buf(data, allow_direct); 1095 } 1096 1097 mt76_dma_rx_fill(dev, q, true); 1098 return done; 1099 } 1100 1101 int mt76_dma_rx_poll(struct napi_struct *napi, int budget) 1102 { 1103 struct mt76_dev *dev; 1104 int qid, done = 0, cur; 1105 1106 dev = mt76_priv(napi->dev); 1107 qid = napi - dev->napi; 1108 1109 rcu_read_lock(); 1110 1111 do { 1112 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done); 1113 mt76_rx_poll_complete(dev, qid, napi); 1114 done += cur; 1115 } while (cur && done < budget); 1116 1117 rcu_read_unlock(); 1118 1119 if (done < budget && napi_complete(napi)) 1120 dev->drv->rx_poll_complete(dev, qid); 1121 1122 return done; 1123 } 1124 EXPORT_SYMBOL_GPL(mt76_dma_rx_poll); 1125 1126 static void 1127 mt76_dma_rx_queue_init(struct mt76_dev *dev, enum mt76_rxq_id qid, 1128 int (*poll)(struct napi_struct *napi, int budget)) 1129 { 1130 netif_napi_add(dev->napi_dev, &dev->napi[qid], poll); 1131 mt76_dma_rx_fill_buf(dev, &dev->q_rx[qid], false); 1132 napi_enable(&dev->napi[qid]); 1133 } 1134 1135 static int 1136 mt76_dma_init(struct mt76_dev *dev, 1137 int (*poll)(struct napi_struct *napi, int budget)) 1138 { 1139 struct mt76_dev **priv; 1140 int i; 1141 1142 dev->napi_dev = alloc_netdev_dummy(sizeof(struct mt76_dev *)); 1143 if (!dev->napi_dev) 1144 return -ENOMEM; 1145 1146 /* napi_dev private data points to mt76_dev parent, so, mt76_dev 1147 * can be retrieved given napi_dev 1148 */ 1149 priv = netdev_priv(dev->napi_dev); 1150 *priv = dev; 1151 1152 dev->tx_napi_dev = alloc_netdev_dummy(sizeof(struct mt76_dev *)); 1153 if (!dev->tx_napi_dev) { 1154 free_netdev(dev->napi_dev); 1155 return -ENOMEM; 1156 } 1157 priv = netdev_priv(dev->tx_napi_dev); 1158 *priv = dev; 1159 1160 snprintf(dev->napi_dev->name, sizeof(dev->napi_dev->name), "%s", 1161 wiphy_name(dev->hw->wiphy)); 1162 dev->napi_dev->threaded = 1; 1163 init_completion(&dev->mmio.wed_reset); 1164 init_completion(&dev->mmio.wed_reset_complete); 1165 1166 mt76_for_each_q_rx(dev, i) { 1167 if (mt76_queue_is_wed_rro(&dev->q_rx[i])) 1168 continue; 1169 1170 mt76_dma_rx_queue_init(dev, i, poll); 1171 } 1172 1173 return 0; 1174 } 1175 1176 static const struct mt76_queue_ops mt76_dma_ops = { 1177 .init = mt76_dma_init, 1178 .alloc = mt76_dma_alloc_queue, 1179 .reset_q = mt76_dma_queue_reset, 1180 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw, 1181 .tx_queue_skb = mt76_dma_tx_queue_skb, 1182 .tx_cleanup = mt76_dma_tx_cleanup, 1183 .rx_queue_init = mt76_dma_rx_queue_init, 1184 .rx_cleanup = mt76_dma_rx_cleanup, 1185 .rx_reset = mt76_dma_rx_reset, 1186 .kick = mt76_dma_kick_queue, 1187 }; 1188 1189 void mt76_dma_attach(struct mt76_dev *dev) 1190 { 1191 dev->queue_ops = &mt76_dma_ops; 1192 } 1193 EXPORT_SYMBOL_GPL(mt76_dma_attach); 1194 1195 void mt76_dma_cleanup(struct mt76_dev *dev) 1196 { 1197 int i; 1198 1199 mt76_worker_disable(&dev->tx_worker); 1200 napi_disable(&dev->tx_napi); 1201 netif_napi_del(&dev->tx_napi); 1202 1203 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 1204 struct mt76_phy *phy = dev->phys[i]; 1205 int j; 1206 1207 if (!phy) 1208 continue; 1209 1210 for (j = 0; j < ARRAY_SIZE(phy->q_tx); j++) 1211 mt76_dma_tx_cleanup(dev, phy->q_tx[j], true); 1212 } 1213 1214 for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++) 1215 mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true); 1216 1217 mt76_for_each_q_rx(dev, i) { 1218 struct mt76_queue *q = &dev->q_rx[i]; 1219 1220 netif_napi_del(&dev->napi[i]); 1221 mt76_dma_rx_cleanup(dev, q); 1222 1223 page_pool_destroy(q->page_pool); 1224 } 1225 1226 if (mtk_wed_device_active(&dev->mmio.wed)) 1227 mtk_wed_device_detach(&dev->mmio.wed); 1228 1229 if (mtk_wed_device_active(&dev->mmio.wed_hif2)) 1230 mtk_wed_device_detach(&dev->mmio.wed_hif2); 1231 1232 mt76_free_pending_txwi(dev); 1233 mt76_free_pending_rxwi(dev); 1234 free_netdev(dev->napi_dev); 1235 free_netdev(dev->tx_napi_dev); 1236 } 1237 EXPORT_SYMBOL_GPL(mt76_dma_cleanup); 1238