1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include "mt76.h" 7 8 static int 9 mt76_txq_get_qid(struct ieee80211_txq *txq) 10 { 11 if (!txq->sta) 12 return MT_TXQ_BE; 13 14 return txq->ac; 15 } 16 17 void 18 mt76_tx_check_agg_ssn(struct ieee80211_sta *sta, struct sk_buff *skb) 19 { 20 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 21 struct ieee80211_txq *txq; 22 struct mt76_txq *mtxq; 23 u8 tid; 24 25 if (!sta || !ieee80211_is_data_qos(hdr->frame_control) || 26 !ieee80211_is_data_present(hdr->frame_control)) 27 return; 28 29 tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; 30 txq = sta->txq[tid]; 31 mtxq = (struct mt76_txq *)txq->drv_priv; 32 if (!mtxq->aggr) 33 return; 34 35 mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10; 36 } 37 EXPORT_SYMBOL_GPL(mt76_tx_check_agg_ssn); 38 39 void 40 mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list) 41 __acquires(&dev->status_lock) 42 { 43 __skb_queue_head_init(list); 44 spin_lock_bh(&dev->status_lock); 45 } 46 EXPORT_SYMBOL_GPL(mt76_tx_status_lock); 47 48 void 49 mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) 50 __releases(&dev->status_lock) 51 { 52 struct ieee80211_hw *hw; 53 struct sk_buff *skb; 54 55 spin_unlock_bh(&dev->status_lock); 56 57 rcu_read_lock(); 58 while ((skb = __skb_dequeue(list)) != NULL) { 59 struct ieee80211_tx_status status = { 60 .skb = skb, 61 .info = IEEE80211_SKB_CB(skb), 62 }; 63 struct ieee80211_rate_status rs = {}; 64 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 65 struct mt76_wcid *wcid; 66 67 wcid = __mt76_wcid_ptr(dev, cb->wcid); 68 if (wcid) { 69 status.sta = wcid_to_sta(wcid); 70 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 71 rs.rate_idx = wcid->rate; 72 status.rates = &rs; 73 status.n_rates = 1; 74 } else { 75 status.n_rates = 0; 76 } 77 } 78 79 hw = mt76_tx_status_get_hw(dev, skb); 80 spin_lock_bh(&dev->rx_lock); 81 ieee80211_tx_status_ext(hw, &status); 82 spin_unlock_bh(&dev->rx_lock); 83 } 84 rcu_read_unlock(); 85 } 86 EXPORT_SYMBOL_GPL(mt76_tx_status_unlock); 87 88 static void 89 __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, 90 struct sk_buff_head *list) 91 { 92 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 93 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 94 u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE; 95 96 flags |= cb->flags; 97 cb->flags = flags; 98 99 if ((flags & done) != done) 100 return; 101 102 /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ 103 if (flags & MT_TX_CB_TXS_FAILED && 104 (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) { 105 info->status.rates[0].count = 0; 106 info->status.rates[0].idx = -1; 107 info->flags |= IEEE80211_TX_STAT_ACK; 108 } 109 110 __skb_queue_tail(list, skb); 111 } 112 113 void 114 mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, 115 struct sk_buff_head *list) 116 { 117 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list); 118 } 119 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done); 120 121 int 122 mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid, 123 struct sk_buff *skb) 124 { 125 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 126 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 127 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 128 int pid; 129 130 memset(cb, 0, sizeof(*cb)); 131 132 if (!wcid) 133 return MT_PACKET_ID_NO_ACK; 134 135 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 136 return MT_PACKET_ID_NO_ACK; 137 138 if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | 139 IEEE80211_TX_CTL_RATE_CTRL_PROBE))) { 140 if (mtk_wed_device_active(&dev->mmio.wed) && 141 ((info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) || 142 ieee80211_is_data(hdr->frame_control))) 143 return MT_PACKET_ID_WED; 144 145 return MT_PACKET_ID_NO_SKB; 146 } 147 148 spin_lock_bh(&dev->status_lock); 149 150 if (rcu_access_pointer(dev->wcid[wcid->idx]) != wcid) { 151 pid = MT_PACKET_ID_NO_ACK; 152 goto out; 153 } 154 155 pid = idr_alloc(&wcid->pktid, skb, MT_PACKET_ID_FIRST, 156 MT_PACKET_ID_MASK, GFP_ATOMIC); 157 if (pid < 0) { 158 pid = MT_PACKET_ID_NO_SKB; 159 goto out; 160 } 161 162 cb->wcid = wcid->idx; 163 cb->pktid = pid; 164 165 if (list_empty(&wcid->list)) 166 list_add_tail(&wcid->list, &dev->wcid_list); 167 168 out: 169 spin_unlock_bh(&dev->status_lock); 170 171 return pid; 172 } 173 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add); 174 175 struct sk_buff * 176 mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid, 177 struct sk_buff_head *list) 178 { 179 struct sk_buff *skb; 180 int id; 181 182 lockdep_assert_held(&dev->status_lock); 183 184 skb = idr_remove(&wcid->pktid, pktid); 185 if (skb) 186 goto out; 187 188 /* look for stale entries in the wcid idr queue */ 189 idr_for_each_entry(&wcid->pktid, skb, id) { 190 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 191 192 if (pktid >= 0) { 193 if (!(cb->flags & MT_TX_CB_DMA_DONE)) 194 continue; 195 196 if (time_is_after_jiffies(cb->jiffies + 197 MT_TX_STATUS_SKB_TIMEOUT)) 198 continue; 199 } 200 201 /* It has been too long since DMA_DONE, time out this packet 202 * and stop waiting for TXS callback. 203 */ 204 idr_remove(&wcid->pktid, cb->pktid); 205 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED | 206 MT_TX_CB_TXS_DONE, list); 207 } 208 209 out: 210 if (idr_is_empty(&wcid->pktid)) 211 list_del_init(&wcid->list); 212 213 return skb; 214 } 215 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get); 216 217 void 218 mt76_tx_status_check(struct mt76_dev *dev, bool flush) 219 { 220 struct mt76_wcid *wcid, *tmp; 221 struct sk_buff_head list; 222 223 mt76_tx_status_lock(dev, &list); 224 list_for_each_entry_safe(wcid, tmp, &dev->wcid_list, list) 225 mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list); 226 mt76_tx_status_unlock(dev, &list); 227 } 228 EXPORT_SYMBOL_GPL(mt76_tx_status_check); 229 230 static void 231 mt76_tx_check_non_aql(struct mt76_dev *dev, struct mt76_wcid *wcid, 232 struct sk_buff *skb) 233 { 234 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 235 struct ieee80211_sta *sta; 236 int pending; 237 int i; 238 239 if (!wcid || info->tx_time_est) 240 return; 241 242 wcid = mt76_wcid_primary(wcid); 243 244 pending = atomic_dec_return(&wcid->non_aql_packets); 245 if (pending < 0) 246 atomic_cmpxchg(&wcid->non_aql_packets, pending, 0); 247 248 sta = wcid_to_sta(wcid); 249 if (!sta || pending != MT_MAX_NON_AQL_PKT - 1) 250 return; 251 252 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 253 if (!sta->txq[i]) 254 continue; 255 256 ieee80211_schedule_txq(dev->hw, sta->txq[i]); 257 } 258 } 259 260 void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb, 261 struct list_head *free_list) 262 { 263 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 264 struct ieee80211_tx_status status = { 265 .skb = skb, 266 .free_list = free_list, 267 }; 268 struct mt76_wcid *wcid = NULL; 269 struct ieee80211_hw *hw; 270 struct sk_buff_head list; 271 272 rcu_read_lock(); 273 274 wcid = __mt76_wcid_ptr(dev, wcid_idx); 275 mt76_tx_check_non_aql(dev, wcid, skb); 276 277 if (wcid && (dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING) && 278 test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 279 struct ieee80211_sta *sta = wcid_to_sta(wcid); 280 281 if (sta) { 282 struct ieee80211_hw *hw = mt76_phy_hw(dev, wcid->phy_idx); 283 int i; 284 285 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) 286 if (sta->txq[i]) 287 ieee80211_schedule_txq(hw, sta->txq[i]); 288 mt76_worker_schedule(&dev->tx_worker); 289 } 290 } 291 292 #ifdef CONFIG_NL80211_TESTMODE 293 if (mt76_is_testmode_skb(dev, skb, &hw)) { 294 struct mt76_phy *phy = hw->priv; 295 296 if (skb == phy->test.tx_skb) 297 phy->test.tx_done++; 298 if (phy->test.tx_queued == phy->test.tx_done) 299 wake_up(&dev->tx_wait); 300 301 dev_kfree_skb_any(skb); 302 goto out; 303 } 304 #endif 305 306 if (cb->pktid < MT_PACKET_ID_FIRST) { 307 struct ieee80211_rate_status rs = {}; 308 309 hw = mt76_tx_status_get_hw(dev, skb); 310 status.sta = wcid_to_sta(wcid); 311 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 312 rs.rate_idx = wcid->rate; 313 status.rates = &rs; 314 status.n_rates = 1; 315 } 316 spin_lock_bh(&dev->rx_lock); 317 ieee80211_tx_status_ext(hw, &status); 318 spin_unlock_bh(&dev->rx_lock); 319 goto out; 320 } 321 322 mt76_tx_status_lock(dev, &list); 323 cb->jiffies = jiffies; 324 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list); 325 mt76_tx_status_unlock(dev, &list); 326 327 out: 328 rcu_read_unlock(); 329 } 330 EXPORT_SYMBOL_GPL(__mt76_tx_complete_skb); 331 332 static int 333 __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, 334 struct mt76_wcid *wcid, struct ieee80211_sta *sta, 335 bool *stop) 336 { 337 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 338 struct mt76_queue *q = phy->q_tx[qid]; 339 struct mt76_dev *dev = phy->dev; 340 bool non_aql; 341 int pending; 342 int idx; 343 344 non_aql = !info->tx_time_est; 345 idx = dev->queue_ops->tx_queue_skb(phy, q, qid, skb, wcid, sta); 346 if (idx < 0 || !sta) 347 return idx; 348 349 q->entry[idx].wcid = wcid->idx; 350 351 if (!non_aql) 352 return idx; 353 354 /* the hardware can report the completion on a different link */ 355 pending = atomic_inc_return(&mt76_wcid_primary(wcid)->non_aql_packets); 356 if (stop && pending >= MT_MAX_NON_AQL_PKT) 357 *stop = true; 358 359 return idx; 360 } 361 362 void 363 mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta, 364 struct mt76_wcid *wcid, struct sk_buff *skb) 365 { 366 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 367 struct ieee80211_hdr *hdr = (void *)skb->data; 368 struct sk_buff_head *head; 369 370 if (mt76_testmode_enabled(phy)) { 371 ieee80211_free_txskb(phy->hw, skb); 372 return; 373 } 374 375 if (WARN_ON(skb_get_queue_mapping(skb) >= MT_TXQ_PSD)) 376 skb_set_queue_mapping(skb, MT_TXQ_BE); 377 378 if (wcid && !(wcid->tx_info & MT_WCID_TX_INFO_SET)) 379 ieee80211_get_tx_rates(info->control.vif, sta, skb, 380 info->control.rates, 1); 381 382 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 383 384 if ((info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) || 385 ((info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK) && 386 ieee80211_is_probe_req(hdr->frame_control))) 387 head = &wcid->tx_offchannel; 388 else 389 head = &wcid->tx_pending; 390 391 spin_lock_bh(&head->lock); 392 __skb_queue_tail(head, skb); 393 spin_unlock_bh(&head->lock); 394 395 spin_lock_bh(&phy->tx_lock); 396 if (list_empty(&wcid->tx_list)) 397 list_add_tail(&wcid->tx_list, &phy->tx_list); 398 spin_unlock_bh(&phy->tx_lock); 399 400 mt76_worker_schedule(&phy->dev->tx_worker); 401 } 402 EXPORT_SYMBOL_GPL(mt76_tx); 403 404 static struct sk_buff * 405 mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq) 406 { 407 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 408 struct ieee80211_tx_info *info; 409 struct sk_buff *skb; 410 411 skb = ieee80211_tx_dequeue(phy->hw, txq); 412 if (!skb) 413 return NULL; 414 415 info = IEEE80211_SKB_CB(skb); 416 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 417 418 return skb; 419 } 420 421 static void 422 mt76_queue_ps_skb(struct mt76_phy *phy, struct ieee80211_sta *sta, 423 struct sk_buff *skb, bool last, 424 enum ieee80211_frame_release_type reason) 425 { 426 struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; 427 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 428 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 429 430 info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE; 431 if (last) { 432 info->flags |= IEEE80211_TX_STATUS_EOSP | 433 IEEE80211_TX_CTL_REQ_TX_STATUS; 434 435 if (reason == IEEE80211_FRAME_RELEASE_UAPSD && 436 ieee80211_is_data_qos(hdr->frame_control)) 437 *ieee80211_get_qos_ctl(hdr) |= IEEE80211_QOS_CTL_EOSP; 438 } 439 440 mt76_skb_set_moredata(skb, !last); 441 __mt76_tx_queue_skb(phy, MT_TXQ_PSD, skb, wcid, sta, NULL); 442 } 443 444 void 445 mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, 446 u16 tids, int nframes, 447 enum ieee80211_frame_release_type reason, 448 bool more_data) 449 { 450 struct mt76_phy *phy = hw->priv; 451 struct mt76_dev *dev = phy->dev; 452 struct sk_buff *last_skb = NULL; 453 struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD]; 454 int i; 455 456 spin_lock_bh(&hwq->lock); 457 for (i = 0; tids && nframes; i++, tids >>= 1) { 458 struct ieee80211_txq *txq = sta->txq[i]; 459 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 460 struct sk_buff *skb; 461 462 if (!(tids & 1)) 463 continue; 464 465 do { 466 skb = mt76_txq_dequeue(phy, mtxq); 467 if (!skb) 468 break; 469 470 nframes--; 471 if (last_skb) 472 mt76_queue_ps_skb(phy, sta, last_skb, false, 473 reason); 474 475 last_skb = skb; 476 } while (nframes); 477 } 478 479 if (last_skb) { 480 mt76_queue_ps_skb(phy, sta, last_skb, true, reason); 481 dev->queue_ops->kick(dev, hwq); 482 } else { 483 ieee80211_sta_eosp(sta); 484 } 485 486 spin_unlock_bh(&hwq->lock); 487 } 488 EXPORT_SYMBOL_GPL(mt76_release_buffered_frames); 489 490 static bool 491 mt76_txq_stopped(struct mt76_queue *q) 492 { 493 return q->stopped || q->blocked || 494 q->queued + MT_TXQ_FREE_THR >= q->ndesc; 495 } 496 497 static int 498 mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, 499 struct mt76_txq *mtxq, struct mt76_wcid *wcid) 500 { 501 struct mt76_dev *dev = phy->dev; 502 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 503 enum mt76_txq_id qid = mt76_txq_get_qid(txq); 504 struct ieee80211_tx_info *info; 505 struct sk_buff *skb; 506 int n_frames = 1; 507 bool stop = false; 508 int idx; 509 510 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 511 if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) 512 return 0; 513 if (ieee80211_txq_aql_pending(phy->hw, txq)) 514 return 0; 515 } 516 517 if (atomic_read(&mt76_wcid_primary(wcid)->non_aql_packets) >= 518 MT_MAX_NON_AQL_PKT) 519 return 0; 520 521 skb = mt76_txq_dequeue(phy, mtxq); 522 if (!skb) 523 return 0; 524 525 info = IEEE80211_SKB_CB(skb); 526 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 527 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 528 info->control.rates, 1); 529 530 spin_lock(&q->lock); 531 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 532 spin_unlock(&q->lock); 533 if (idx < 0) 534 return idx; 535 536 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 537 goto out; 538 539 do { 540 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 541 break; 542 543 if (stop || mt76_txq_stopped(q)) 544 break; 545 546 skb = mt76_txq_dequeue(phy, mtxq); 547 if (!skb) 548 break; 549 550 info = IEEE80211_SKB_CB(skb); 551 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 552 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 553 info->control.rates, 1); 554 555 spin_lock(&q->lock); 556 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 557 spin_unlock(&q->lock); 558 if (idx < 0) 559 break; 560 561 n_frames++; 562 } while (1); 563 564 out: 565 spin_lock(&q->lock); 566 dev->queue_ops->kick(dev, q); 567 spin_unlock(&q->lock); 568 569 return n_frames; 570 } 571 572 static int 573 mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) 574 { 575 struct mt76_dev *dev = phy->dev; 576 struct ieee80211_txq *txq; 577 struct mt76_txq *mtxq; 578 struct mt76_wcid *wcid; 579 struct mt76_queue *q; 580 int ret = 0; 581 582 while (1) { 583 int n_frames = 0; 584 585 txq = ieee80211_next_txq(phy->hw, qid); 586 if (!txq) 587 break; 588 589 mtxq = (struct mt76_txq *)txq->drv_priv; 590 wcid = __mt76_wcid_ptr(dev, mtxq->wcid); 591 if (!wcid) 592 continue; 593 594 phy = mt76_dev_phy(dev, wcid->phy_idx); 595 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 596 continue; 597 598 q = phy->q_tx[qid]; 599 600 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 601 if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) 602 continue; 603 604 if (!mt76_txq_stopped(q)) 605 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 606 607 ieee80211_return_txq(phy->hw, txq, false); 608 609 if (unlikely(n_frames < 0)) 610 return n_frames; 611 ret += n_frames; 612 continue; 613 } 614 615 if (atomic_read(&mt76_wcid_primary(wcid)->non_aql_packets) >= 616 MT_MAX_NON_AQL_PKT) 617 continue; 618 if (dev->queue_ops->tx_cleanup && 619 q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { 620 dev->queue_ops->tx_cleanup(dev, q, false); 621 } 622 623 if (mtxq->send_bar && mtxq->aggr) { 624 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 625 struct ieee80211_sta *sta = txq->sta; 626 struct ieee80211_vif *vif = txq->vif; 627 u16 agg_ssn = mtxq->agg_ssn; 628 u8 tid = txq->tid; 629 630 mtxq->send_bar = false; 631 ieee80211_send_bar(vif, sta->addr, tid, agg_ssn); 632 } 633 634 if (!mt76_txq_stopped(q)) 635 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 636 637 ieee80211_return_txq(phy->hw, txq, false); 638 639 if (unlikely(n_frames < 0)) 640 return n_frames; 641 642 ret += n_frames; 643 } 644 645 return ret; 646 } 647 648 void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid) 649 { 650 int len; 651 652 if (qid >= 4) 653 return; 654 655 local_bh_disable(); 656 rcu_read_lock(); 657 658 do { 659 ieee80211_txq_schedule_start(phy->hw, qid); 660 len = mt76_txq_schedule_list(phy, qid); 661 ieee80211_txq_schedule_end(phy->hw, qid); 662 } while (len > 0); 663 664 rcu_read_unlock(); 665 local_bh_enable(); 666 } 667 EXPORT_SYMBOL_GPL(mt76_txq_schedule); 668 669 static int 670 mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, 671 struct sk_buff_head *head) 672 { 673 struct mt76_dev *dev = phy->dev; 674 struct ieee80211_sta *sta; 675 struct mt76_queue *q; 676 struct sk_buff *skb; 677 int ret = 0; 678 679 spin_lock(&head->lock); 680 while ((skb = skb_peek(head)) != NULL) { 681 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 682 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 683 int qid = skb_get_queue_mapping(skb); 684 685 if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) && 686 !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && 687 !ieee80211_is_data_present(hdr->frame_control) && 688 (!(wcid->tx_info & MT_WCID_TX_INFO_SET) || 689 !ieee80211_is_bufferable_mmpdu(skb) || 690 ieee80211_is_deauth(hdr->frame_control) || 691 ieee80211_is_disassoc(hdr->frame_control) || 692 head == &wcid->tx_offchannel)) 693 qid = MT_TXQ_PSD; 694 695 q = phy->q_tx[qid]; 696 if (mt76_txq_stopped(q) || test_bit(MT76_RESET, &phy->state)) { 697 ret = -1; 698 break; 699 } 700 701 __skb_unlink(skb, head); 702 spin_unlock(&head->lock); 703 704 sta = wcid_to_sta(wcid); 705 spin_lock(&q->lock); 706 __mt76_tx_queue_skb(phy, qid, skb, wcid, sta, NULL); 707 dev->queue_ops->kick(dev, q); 708 spin_unlock(&q->lock); 709 710 spin_lock(&head->lock); 711 } 712 spin_unlock(&head->lock); 713 714 return ret; 715 } 716 717 void mt76_txq_schedule_pending(struct mt76_phy *phy) 718 { 719 LIST_HEAD(tx_list); 720 int ret = 0; 721 722 if (list_empty(&phy->tx_list)) 723 return; 724 725 local_bh_disable(); 726 rcu_read_lock(); 727 728 spin_lock(&phy->tx_lock); 729 list_splice_init(&phy->tx_list, &tx_list); 730 while (!list_empty(&tx_list)) { 731 struct mt76_wcid *wcid; 732 733 wcid = list_first_entry(&tx_list, struct mt76_wcid, tx_list); 734 list_del_init(&wcid->tx_list); 735 736 spin_unlock(&phy->tx_lock); 737 if (ret >= 0) 738 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel); 739 if (ret >= 0 && !phy->offchannel) 740 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); 741 spin_lock(&phy->tx_lock); 742 743 if ((!skb_queue_empty(&wcid->tx_pending) || 744 !skb_queue_empty(&wcid->tx_offchannel)) && 745 list_empty(&wcid->tx_list)) 746 list_add_tail(&wcid->tx_list, &phy->tx_list); 747 } 748 spin_unlock(&phy->tx_lock); 749 750 rcu_read_unlock(); 751 local_bh_enable(); 752 } 753 754 void mt76_txq_schedule_all(struct mt76_phy *phy) 755 { 756 struct mt76_phy *main_phy = &phy->dev->phy; 757 int i; 758 759 mt76_txq_schedule_pending(phy); 760 761 if (phy != main_phy && phy->hw == main_phy->hw) 762 return; 763 764 for (i = 0; i <= MT_TXQ_BK; i++) 765 mt76_txq_schedule(phy, i); 766 } 767 EXPORT_SYMBOL_GPL(mt76_txq_schedule_all); 768 769 void mt76_tx_worker_run(struct mt76_dev *dev) 770 { 771 struct mt76_phy *phy; 772 int i; 773 774 mt76_txq_schedule_all(&dev->phy); 775 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 776 phy = dev->phys[i]; 777 if (!phy) 778 continue; 779 780 mt76_txq_schedule_all(phy); 781 } 782 783 #ifdef CONFIG_NL80211_TESTMODE 784 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 785 phy = dev->phys[i]; 786 if (!phy || !phy->test.tx_pending) 787 continue; 788 789 mt76_testmode_tx_pending(phy); 790 } 791 #endif 792 } 793 EXPORT_SYMBOL_GPL(mt76_tx_worker_run); 794 795 void mt76_tx_worker(struct mt76_worker *w) 796 { 797 struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker); 798 799 mt76_tx_worker_run(dev); 800 } 801 802 void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta, 803 bool send_bar) 804 { 805 int i; 806 807 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 808 struct ieee80211_txq *txq = sta->txq[i]; 809 struct mt76_queue *hwq; 810 struct mt76_txq *mtxq; 811 812 if (!txq) 813 continue; 814 815 hwq = phy->q_tx[mt76_txq_get_qid(txq)]; 816 mtxq = (struct mt76_txq *)txq->drv_priv; 817 818 spin_lock_bh(&hwq->lock); 819 mtxq->send_bar = mtxq->aggr && send_bar; 820 spin_unlock_bh(&hwq->lock); 821 } 822 } 823 EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); 824 825 void mt76_sta_ps_transition(struct mt76_dev *dev, struct mt76_wcid *wcid, 826 bool ps) 827 { 828 struct ieee80211_sta *sta; 829 struct ieee80211_hw *hw; 830 int i; 831 832 if (ps) { 833 set_bit(MT_WCID_FLAG_PS, &wcid->flags); 834 mt76_worker_schedule(&dev->tx_worker); 835 return; 836 } 837 838 clear_bit(MT_WCID_FLAG_PS, &wcid->flags); 839 840 sta = wcid_to_sta(wcid); 841 if (!sta) 842 return; 843 844 hw = mt76_phy_hw(dev, wcid->phy_idx); 845 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) 846 if (sta->txq[i]) 847 ieee80211_schedule_txq(hw, sta->txq[i]); 848 mt76_worker_schedule(&dev->tx_worker); 849 } 850 EXPORT_SYMBOL_GPL(mt76_sta_ps_transition); 851 852 void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) 853 { 854 struct mt76_phy *phy = hw->priv; 855 struct mt76_dev *dev = phy->dev; 856 857 mt76_worker_schedule(&dev->tx_worker); 858 } 859 EXPORT_SYMBOL_GPL(mt76_wake_tx_queue); 860 861 u8 mt76_ac_to_hwq(u8 ac) 862 { 863 static const u8 wmm_queue_map[] = { 864 [IEEE80211_AC_BE] = 0, 865 [IEEE80211_AC_BK] = 1, 866 [IEEE80211_AC_VI] = 2, 867 [IEEE80211_AC_VO] = 3, 868 }; 869 870 if (WARN_ON(ac >= IEEE80211_NUM_ACS)) 871 return 0; 872 873 return wmm_queue_map[ac]; 874 } 875 EXPORT_SYMBOL_GPL(mt76_ac_to_hwq); 876 877 int mt76_skb_adjust_pad(struct sk_buff *skb, int pad) 878 { 879 struct sk_buff *iter, *last = skb; 880 881 /* First packet of a A-MSDU burst keeps track of the whole burst 882 * length, need to update length of it and the last packet. 883 */ 884 skb_walk_frags(skb, iter) { 885 last = iter; 886 if (!iter->next) { 887 skb->data_len += pad; 888 skb->len += pad; 889 break; 890 } 891 } 892 893 if (__skb_pad(last, pad, false)) 894 return -ENOMEM; 895 896 __skb_put(last, pad); 897 898 return 0; 899 } 900 EXPORT_SYMBOL_GPL(mt76_skb_adjust_pad); 901 902 void mt76_queue_tx_complete(struct mt76_dev *dev, struct mt76_queue *q, 903 struct mt76_queue_entry *e) 904 { 905 if (e->skb) 906 dev->drv->tx_complete_skb(dev, e); 907 908 spin_lock_bh(&q->lock); 909 q->tail = (q->tail + 1) % q->ndesc; 910 q->queued--; 911 spin_unlock_bh(&q->lock); 912 } 913 EXPORT_SYMBOL_GPL(mt76_queue_tx_complete); 914 915 void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked) 916 { 917 struct mt76_phy *phy = &dev->phy; 918 struct mt76_queue *q = phy->q_tx[0]; 919 920 if (blocked == q->blocked) 921 return; 922 923 q->blocked = blocked; 924 925 phy = dev->phys[MT_BAND1]; 926 if (phy) { 927 q = phy->q_tx[0]; 928 q->blocked = blocked; 929 } 930 phy = dev->phys[MT_BAND2]; 931 if (phy) { 932 q = phy->q_tx[0]; 933 q->blocked = blocked; 934 } 935 936 if (!blocked) 937 mt76_worker_schedule(&dev->tx_worker); 938 } 939 EXPORT_SYMBOL_GPL(__mt76_set_tx_blocked); 940 941 int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) 942 { 943 int token; 944 945 spin_lock_bh(&dev->token_lock); 946 947 token = idr_alloc(&dev->token, *ptxwi, dev->token_start, 948 dev->token_start + dev->token_size, 949 GFP_ATOMIC); 950 if (token >= dev->token_start) { 951 dev->token_count++; 952 953 if ((*ptxwi)->qid == MT_TXQ_PSD) { 954 struct mt76_phy *mphy = mt76_dev_phy(dev, (*ptxwi)->phy_idx); 955 atomic_inc(&mphy->mgmt_tx_pending); 956 } 957 } 958 959 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 960 if (mtk_wed_device_active(&dev->mmio.wed) && 961 token >= dev->mmio.wed.wlan.token_start) 962 dev->wed_token_count++; 963 #endif 964 965 if (dev->token_count >= dev->token_size - MT76_TOKEN_FREE_THR) 966 __mt76_set_tx_blocked(dev, true); 967 968 spin_unlock_bh(&dev->token_lock); 969 970 return token; 971 } 972 EXPORT_SYMBOL_GPL(mt76_token_consume); 973 974 int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr, 975 struct mt76_txwi_cache *t, dma_addr_t phys) 976 { 977 int token; 978 979 spin_lock_bh(&dev->rx_token_lock); 980 token = idr_alloc(&dev->rx_token, t, 0, dev->rx_token_size, 981 GFP_ATOMIC); 982 if (token >= 0) { 983 t->ptr = ptr; 984 t->dma_addr = phys; 985 } 986 spin_unlock_bh(&dev->rx_token_lock); 987 988 return token; 989 } 990 EXPORT_SYMBOL_GPL(mt76_rx_token_consume); 991 992 struct mt76_txwi_cache * 993 mt76_token_release(struct mt76_dev *dev, int token, bool *wake) 994 { 995 struct mt76_txwi_cache *txwi; 996 997 spin_lock_bh(&dev->token_lock); 998 999 txwi = idr_remove(&dev->token, token); 1000 if (txwi) { 1001 dev->token_count--; 1002 1003 if (txwi->qid == MT_TXQ_PSD) { 1004 struct mt76_phy *mphy = mt76_dev_phy(dev, txwi->phy_idx); 1005 if (atomic_dec_and_test(&mphy->mgmt_tx_pending)) 1006 wake_up(&dev->tx_wait); 1007 } 1008 1009 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 1010 if (mtk_wed_device_active(&dev->mmio.wed) && 1011 token >= dev->mmio.wed.wlan.token_start && 1012 --dev->wed_token_count == 0) 1013 wake_up(&dev->tx_wait); 1014 #endif 1015 } 1016 1017 if (wake && dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && 1018 dev->phy.q_tx[0]->blocked) 1019 *wake = true; 1020 1021 spin_unlock_bh(&dev->token_lock); 1022 1023 return txwi; 1024 } 1025 EXPORT_SYMBOL_GPL(mt76_token_release); 1026 1027 struct mt76_txwi_cache * 1028 mt76_rx_token_release(struct mt76_dev *dev, int token) 1029 { 1030 struct mt76_txwi_cache *t; 1031 1032 spin_lock_bh(&dev->rx_token_lock); 1033 t = idr_remove(&dev->rx_token, token); 1034 spin_unlock_bh(&dev->rx_token_lock); 1035 1036 return t; 1037 } 1038 EXPORT_SYMBOL_GPL(mt76_rx_token_release); 1039