1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include "mt76.h" 7 8 static int 9 mt76_txq_get_qid(struct ieee80211_txq *txq) 10 { 11 if (!txq->sta) 12 return MT_TXQ_BE; 13 14 return txq->ac; 15 } 16 17 void 18 mt76_tx_check_agg_ssn(struct ieee80211_sta *sta, struct sk_buff *skb) 19 { 20 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 21 struct ieee80211_txq *txq; 22 struct mt76_txq *mtxq; 23 u8 tid; 24 25 if (!sta || !ieee80211_is_data_qos(hdr->frame_control) || 26 !ieee80211_is_data_present(hdr->frame_control)) 27 return; 28 29 tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; 30 txq = sta->txq[tid]; 31 mtxq = (struct mt76_txq *)txq->drv_priv; 32 if (!mtxq->aggr) 33 return; 34 35 mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10; 36 } 37 EXPORT_SYMBOL_GPL(mt76_tx_check_agg_ssn); 38 39 void 40 mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list) 41 __acquires(&dev->status_lock) 42 { 43 __skb_queue_head_init(list); 44 spin_lock_bh(&dev->status_lock); 45 } 46 EXPORT_SYMBOL_GPL(mt76_tx_status_lock); 47 48 void 49 mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) 50 __releases(&dev->status_lock) 51 { 52 struct ieee80211_hw *hw; 53 struct sk_buff *skb; 54 55 spin_unlock_bh(&dev->status_lock); 56 57 rcu_read_lock(); 58 while ((skb = __skb_dequeue(list)) != NULL) { 59 struct ieee80211_tx_status status = { 60 .skb = skb, 61 .info = IEEE80211_SKB_CB(skb), 62 }; 63 struct ieee80211_rate_status rs = {}; 64 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 65 struct mt76_wcid *wcid; 66 67 wcid = __mt76_wcid_ptr(dev, cb->wcid); 68 if (wcid) { 69 status.sta = wcid_to_sta(wcid); 70 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 71 rs.rate_idx = wcid->rate; 72 status.rates = &rs; 73 status.n_rates = 1; 74 } else { 75 status.n_rates = 0; 76 } 77 } 78 79 hw = mt76_tx_status_get_hw(dev, skb); 80 spin_lock_bh(&dev->rx_lock); 81 ieee80211_tx_status_ext(hw, &status); 82 spin_unlock_bh(&dev->rx_lock); 83 } 84 rcu_read_unlock(); 85 } 86 EXPORT_SYMBOL_GPL(mt76_tx_status_unlock); 87 88 static void 89 __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, 90 struct sk_buff_head *list) 91 { 92 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 93 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 94 u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE; 95 96 flags |= cb->flags; 97 cb->flags = flags; 98 99 if ((flags & done) != done) 100 return; 101 102 /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ 103 if (flags & MT_TX_CB_TXS_FAILED && 104 (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) { 105 info->status.rates[0].count = 0; 106 info->status.rates[0].idx = -1; 107 info->flags |= IEEE80211_TX_STAT_ACK; 108 } 109 110 __skb_queue_tail(list, skb); 111 } 112 113 void 114 mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, 115 struct sk_buff_head *list) 116 { 117 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list); 118 } 119 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done); 120 121 int 122 mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid, 123 struct sk_buff *skb) 124 { 125 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 126 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 127 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 128 int pid; 129 130 memset(cb, 0, sizeof(*cb)); 131 132 if (!wcid || !rcu_access_pointer(dev->wcid[wcid->idx])) 133 return MT_PACKET_ID_NO_ACK; 134 135 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 136 return MT_PACKET_ID_NO_ACK; 137 138 if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | 139 IEEE80211_TX_CTL_RATE_CTRL_PROBE))) { 140 if (mtk_wed_device_active(&dev->mmio.wed) && 141 ((info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) || 142 ieee80211_is_data(hdr->frame_control))) 143 return MT_PACKET_ID_WED; 144 145 return MT_PACKET_ID_NO_SKB; 146 } 147 148 spin_lock_bh(&dev->status_lock); 149 150 pid = idr_alloc(&wcid->pktid, skb, MT_PACKET_ID_FIRST, 151 MT_PACKET_ID_MASK, GFP_ATOMIC); 152 if (pid < 0) { 153 pid = MT_PACKET_ID_NO_SKB; 154 goto out; 155 } 156 157 cb->wcid = wcid->idx; 158 cb->pktid = pid; 159 160 if (list_empty(&wcid->list)) 161 list_add_tail(&wcid->list, &dev->wcid_list); 162 163 out: 164 spin_unlock_bh(&dev->status_lock); 165 166 return pid; 167 } 168 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add); 169 170 struct sk_buff * 171 mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid, 172 struct sk_buff_head *list) 173 { 174 struct sk_buff *skb; 175 int id; 176 177 lockdep_assert_held(&dev->status_lock); 178 179 skb = idr_remove(&wcid->pktid, pktid); 180 if (skb) 181 goto out; 182 183 /* look for stale entries in the wcid idr queue */ 184 idr_for_each_entry(&wcid->pktid, skb, id) { 185 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 186 187 if (pktid >= 0) { 188 if (!(cb->flags & MT_TX_CB_DMA_DONE)) 189 continue; 190 191 if (time_is_after_jiffies(cb->jiffies + 192 MT_TX_STATUS_SKB_TIMEOUT)) 193 continue; 194 } 195 196 /* It has been too long since DMA_DONE, time out this packet 197 * and stop waiting for TXS callback. 198 */ 199 idr_remove(&wcid->pktid, cb->pktid); 200 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED | 201 MT_TX_CB_TXS_DONE, list); 202 } 203 204 out: 205 if (idr_is_empty(&wcid->pktid)) 206 list_del_init(&wcid->list); 207 208 return skb; 209 } 210 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get); 211 212 void 213 mt76_tx_status_check(struct mt76_dev *dev, bool flush) 214 { 215 struct mt76_wcid *wcid, *tmp; 216 struct sk_buff_head list; 217 218 mt76_tx_status_lock(dev, &list); 219 list_for_each_entry_safe(wcid, tmp, &dev->wcid_list, list) 220 mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list); 221 mt76_tx_status_unlock(dev, &list); 222 } 223 EXPORT_SYMBOL_GPL(mt76_tx_status_check); 224 225 static void 226 mt76_tx_check_non_aql(struct mt76_dev *dev, struct mt76_wcid *wcid, 227 struct sk_buff *skb) 228 { 229 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 230 struct ieee80211_sta *sta; 231 int pending; 232 int i; 233 234 if (!wcid || info->tx_time_est) 235 return; 236 237 pending = atomic_dec_return(&wcid->non_aql_packets); 238 if (pending < 0) 239 atomic_cmpxchg(&wcid->non_aql_packets, pending, 0); 240 241 sta = wcid_to_sta(wcid); 242 if (!sta || pending != MT_MAX_NON_AQL_PKT - 1) 243 return; 244 245 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 246 if (!sta->txq[i]) 247 continue; 248 249 ieee80211_schedule_txq(dev->hw, sta->txq[i]); 250 } 251 } 252 253 void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb, 254 struct list_head *free_list) 255 { 256 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 257 struct ieee80211_tx_status status = { 258 .skb = skb, 259 .free_list = free_list, 260 }; 261 struct mt76_wcid *wcid = NULL; 262 struct ieee80211_hw *hw; 263 struct sk_buff_head list; 264 265 rcu_read_lock(); 266 267 wcid = __mt76_wcid_ptr(dev, wcid_idx); 268 mt76_tx_check_non_aql(dev, wcid, skb); 269 270 if (wcid && (dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING) && 271 test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 272 struct ieee80211_sta *sta = wcid_to_sta(wcid); 273 274 if (sta) { 275 struct ieee80211_hw *hw = mt76_phy_hw(dev, wcid->phy_idx); 276 int i; 277 278 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) 279 if (sta->txq[i]) 280 ieee80211_schedule_txq(hw, sta->txq[i]); 281 mt76_worker_schedule(&dev->tx_worker); 282 } 283 } 284 285 #ifdef CONFIG_NL80211_TESTMODE 286 if (mt76_is_testmode_skb(dev, skb, &hw)) { 287 struct mt76_phy *phy = hw->priv; 288 289 if (skb == phy->test.tx_skb) 290 phy->test.tx_done++; 291 if (phy->test.tx_queued == phy->test.tx_done) 292 wake_up(&dev->tx_wait); 293 294 dev_kfree_skb_any(skb); 295 goto out; 296 } 297 #endif 298 299 if (cb->pktid < MT_PACKET_ID_FIRST) { 300 struct ieee80211_rate_status rs = {}; 301 302 hw = mt76_tx_status_get_hw(dev, skb); 303 status.sta = wcid_to_sta(wcid); 304 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 305 rs.rate_idx = wcid->rate; 306 status.rates = &rs; 307 status.n_rates = 1; 308 } 309 spin_lock_bh(&dev->rx_lock); 310 ieee80211_tx_status_ext(hw, &status); 311 spin_unlock_bh(&dev->rx_lock); 312 goto out; 313 } 314 315 mt76_tx_status_lock(dev, &list); 316 cb->jiffies = jiffies; 317 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list); 318 mt76_tx_status_unlock(dev, &list); 319 320 out: 321 rcu_read_unlock(); 322 } 323 EXPORT_SYMBOL_GPL(__mt76_tx_complete_skb); 324 325 static int 326 __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, 327 struct mt76_wcid *wcid, struct ieee80211_sta *sta, 328 bool *stop) 329 { 330 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 331 struct mt76_queue *q = phy->q_tx[qid]; 332 struct mt76_dev *dev = phy->dev; 333 bool non_aql; 334 int pending; 335 int idx; 336 337 non_aql = !info->tx_time_est; 338 idx = dev->queue_ops->tx_queue_skb(phy, q, qid, skb, wcid, sta); 339 if (idx < 0 || !sta) 340 return idx; 341 342 q->entry[idx].wcid = wcid->idx; 343 344 if (!non_aql) 345 return idx; 346 347 pending = atomic_inc_return(&wcid->non_aql_packets); 348 if (stop && pending >= MT_MAX_NON_AQL_PKT) 349 *stop = true; 350 351 return idx; 352 } 353 354 void 355 mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta, 356 struct mt76_wcid *wcid, struct sk_buff *skb) 357 { 358 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 359 struct ieee80211_hdr *hdr = (void *)skb->data; 360 struct sk_buff_head *head; 361 362 if (mt76_testmode_enabled(phy)) { 363 ieee80211_free_txskb(phy->hw, skb); 364 return; 365 } 366 367 if (WARN_ON(skb_get_queue_mapping(skb) >= MT_TXQ_PSD)) 368 skb_set_queue_mapping(skb, MT_TXQ_BE); 369 370 if (wcid && !(wcid->tx_info & MT_WCID_TX_INFO_SET)) 371 ieee80211_get_tx_rates(info->control.vif, sta, skb, 372 info->control.rates, 1); 373 374 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 375 376 if ((info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) || 377 ((info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK) && 378 ieee80211_is_probe_req(hdr->frame_control))) 379 head = &wcid->tx_offchannel; 380 else 381 head = &wcid->tx_pending; 382 383 spin_lock_bh(&head->lock); 384 __skb_queue_tail(head, skb); 385 spin_unlock_bh(&head->lock); 386 387 spin_lock_bh(&phy->tx_lock); 388 if (list_empty(&wcid->tx_list)) 389 list_add_tail(&wcid->tx_list, &phy->tx_list); 390 spin_unlock_bh(&phy->tx_lock); 391 392 mt76_worker_schedule(&phy->dev->tx_worker); 393 } 394 EXPORT_SYMBOL_GPL(mt76_tx); 395 396 static struct sk_buff * 397 mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq) 398 { 399 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 400 struct ieee80211_tx_info *info; 401 struct sk_buff *skb; 402 403 skb = ieee80211_tx_dequeue(phy->hw, txq); 404 if (!skb) 405 return NULL; 406 407 info = IEEE80211_SKB_CB(skb); 408 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 409 410 return skb; 411 } 412 413 static void 414 mt76_queue_ps_skb(struct mt76_phy *phy, struct ieee80211_sta *sta, 415 struct sk_buff *skb, bool last, 416 enum ieee80211_frame_release_type reason) 417 { 418 struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; 419 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 420 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 421 422 info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE; 423 if (last) { 424 info->flags |= IEEE80211_TX_STATUS_EOSP | 425 IEEE80211_TX_CTL_REQ_TX_STATUS; 426 427 if (reason == IEEE80211_FRAME_RELEASE_UAPSD && 428 ieee80211_is_data_qos(hdr->frame_control)) 429 *ieee80211_get_qos_ctl(hdr) |= IEEE80211_QOS_CTL_EOSP; 430 } 431 432 mt76_skb_set_moredata(skb, !last); 433 __mt76_tx_queue_skb(phy, MT_TXQ_PSD, skb, wcid, sta, NULL); 434 } 435 436 void 437 mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, 438 u16 tids, int nframes, 439 enum ieee80211_frame_release_type reason, 440 bool more_data) 441 { 442 struct mt76_phy *phy = hw->priv; 443 struct mt76_dev *dev = phy->dev; 444 struct sk_buff *last_skb = NULL; 445 struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD]; 446 int i; 447 448 spin_lock_bh(&hwq->lock); 449 for (i = 0; tids && nframes; i++, tids >>= 1) { 450 struct ieee80211_txq *txq = sta->txq[i]; 451 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 452 struct sk_buff *skb; 453 454 if (!(tids & 1)) 455 continue; 456 457 do { 458 skb = mt76_txq_dequeue(phy, mtxq); 459 if (!skb) 460 break; 461 462 nframes--; 463 if (last_skb) 464 mt76_queue_ps_skb(phy, sta, last_skb, false, 465 reason); 466 467 last_skb = skb; 468 } while (nframes); 469 } 470 471 if (last_skb) { 472 mt76_queue_ps_skb(phy, sta, last_skb, true, reason); 473 dev->queue_ops->kick(dev, hwq); 474 } else { 475 ieee80211_sta_eosp(sta); 476 } 477 478 spin_unlock_bh(&hwq->lock); 479 } 480 EXPORT_SYMBOL_GPL(mt76_release_buffered_frames); 481 482 static bool 483 mt76_txq_stopped(struct mt76_queue *q) 484 { 485 return q->stopped || q->blocked || 486 q->queued + MT_TXQ_FREE_THR >= q->ndesc; 487 } 488 489 static int 490 mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, 491 struct mt76_txq *mtxq, struct mt76_wcid *wcid) 492 { 493 struct mt76_dev *dev = phy->dev; 494 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 495 enum mt76_txq_id qid = mt76_txq_get_qid(txq); 496 struct ieee80211_tx_info *info; 497 struct sk_buff *skb; 498 int n_frames = 1; 499 bool stop = false; 500 int idx; 501 502 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 503 if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) 504 return 0; 505 if (ieee80211_txq_aql_pending(phy->hw, txq)) 506 return 0; 507 } 508 509 if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) 510 return 0; 511 512 skb = mt76_txq_dequeue(phy, mtxq); 513 if (!skb) 514 return 0; 515 516 info = IEEE80211_SKB_CB(skb); 517 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 518 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 519 info->control.rates, 1); 520 521 spin_lock(&q->lock); 522 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 523 spin_unlock(&q->lock); 524 if (idx < 0) 525 return idx; 526 527 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 528 goto out; 529 530 do { 531 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 532 break; 533 534 if (stop || mt76_txq_stopped(q)) 535 break; 536 537 skb = mt76_txq_dequeue(phy, mtxq); 538 if (!skb) 539 break; 540 541 info = IEEE80211_SKB_CB(skb); 542 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 543 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 544 info->control.rates, 1); 545 546 spin_lock(&q->lock); 547 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 548 spin_unlock(&q->lock); 549 if (idx < 0) 550 break; 551 552 n_frames++; 553 } while (1); 554 555 out: 556 spin_lock(&q->lock); 557 dev->queue_ops->kick(dev, q); 558 spin_unlock(&q->lock); 559 560 return n_frames; 561 } 562 563 static int 564 mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) 565 { 566 struct mt76_dev *dev = phy->dev; 567 struct ieee80211_txq *txq; 568 struct mt76_txq *mtxq; 569 struct mt76_wcid *wcid; 570 struct mt76_queue *q; 571 int ret = 0; 572 573 while (1) { 574 int n_frames = 0; 575 576 txq = ieee80211_next_txq(phy->hw, qid); 577 if (!txq) 578 break; 579 580 mtxq = (struct mt76_txq *)txq->drv_priv; 581 wcid = __mt76_wcid_ptr(dev, mtxq->wcid); 582 if (!wcid) 583 continue; 584 585 phy = mt76_dev_phy(dev, wcid->phy_idx); 586 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 587 continue; 588 589 q = phy->q_tx[qid]; 590 591 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 592 if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) 593 continue; 594 595 if (!mt76_txq_stopped(q)) 596 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 597 598 ieee80211_return_txq(phy->hw, txq, false); 599 600 if (unlikely(n_frames < 0)) 601 return n_frames; 602 ret += n_frames; 603 continue; 604 } 605 606 if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) 607 continue; 608 if (dev->queue_ops->tx_cleanup && 609 q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { 610 dev->queue_ops->tx_cleanup(dev, q, false); 611 } 612 613 if (mtxq->send_bar && mtxq->aggr) { 614 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 615 struct ieee80211_sta *sta = txq->sta; 616 struct ieee80211_vif *vif = txq->vif; 617 u16 agg_ssn = mtxq->agg_ssn; 618 u8 tid = txq->tid; 619 620 mtxq->send_bar = false; 621 ieee80211_send_bar(vif, sta->addr, tid, agg_ssn); 622 } 623 624 if (!mt76_txq_stopped(q)) 625 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 626 627 ieee80211_return_txq(phy->hw, txq, false); 628 629 if (unlikely(n_frames < 0)) 630 return n_frames; 631 632 ret += n_frames; 633 } 634 635 return ret; 636 } 637 638 void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid) 639 { 640 int len; 641 642 if (qid >= 4) 643 return; 644 645 local_bh_disable(); 646 rcu_read_lock(); 647 648 do { 649 ieee80211_txq_schedule_start(phy->hw, qid); 650 len = mt76_txq_schedule_list(phy, qid); 651 ieee80211_txq_schedule_end(phy->hw, qid); 652 } while (len > 0); 653 654 rcu_read_unlock(); 655 local_bh_enable(); 656 } 657 EXPORT_SYMBOL_GPL(mt76_txq_schedule); 658 659 static int 660 mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, 661 struct sk_buff_head *head) 662 { 663 struct mt76_dev *dev = phy->dev; 664 struct ieee80211_sta *sta; 665 struct mt76_queue *q; 666 struct sk_buff *skb; 667 int ret = 0; 668 669 spin_lock(&head->lock); 670 while ((skb = skb_peek(head)) != NULL) { 671 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 672 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 673 int qid = skb_get_queue_mapping(skb); 674 675 if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) && 676 !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && 677 !ieee80211_is_data_present(hdr->frame_control) && 678 (!ieee80211_is_bufferable_mmpdu(skb) || 679 ieee80211_is_deauth(hdr->frame_control) || 680 ieee80211_is_disassoc(hdr->frame_control) || 681 head == &wcid->tx_offchannel)) 682 qid = MT_TXQ_PSD; 683 684 q = phy->q_tx[qid]; 685 if (mt76_txq_stopped(q) || test_bit(MT76_RESET, &phy->state)) { 686 ret = -1; 687 break; 688 } 689 690 __skb_unlink(skb, head); 691 spin_unlock(&head->lock); 692 693 sta = wcid_to_sta(wcid); 694 spin_lock(&q->lock); 695 __mt76_tx_queue_skb(phy, qid, skb, wcid, sta, NULL); 696 dev->queue_ops->kick(dev, q); 697 spin_unlock(&q->lock); 698 699 spin_lock(&head->lock); 700 } 701 spin_unlock(&head->lock); 702 703 return ret; 704 } 705 706 void mt76_txq_schedule_pending(struct mt76_phy *phy) 707 { 708 LIST_HEAD(tx_list); 709 int ret = 0; 710 711 if (list_empty(&phy->tx_list)) 712 return; 713 714 local_bh_disable(); 715 rcu_read_lock(); 716 717 spin_lock(&phy->tx_lock); 718 list_splice_init(&phy->tx_list, &tx_list); 719 while (!list_empty(&tx_list)) { 720 struct mt76_wcid *wcid; 721 722 wcid = list_first_entry(&tx_list, struct mt76_wcid, tx_list); 723 list_del_init(&wcid->tx_list); 724 725 spin_unlock(&phy->tx_lock); 726 if (ret >= 0) 727 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel); 728 if (ret >= 0 && !phy->offchannel) 729 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); 730 spin_lock(&phy->tx_lock); 731 732 if ((!skb_queue_empty(&wcid->tx_pending) || 733 !skb_queue_empty(&wcid->tx_offchannel)) && 734 list_empty(&wcid->tx_list)) 735 list_add_tail(&wcid->tx_list, &phy->tx_list); 736 } 737 spin_unlock(&phy->tx_lock); 738 739 rcu_read_unlock(); 740 local_bh_enable(); 741 } 742 743 void mt76_txq_schedule_all(struct mt76_phy *phy) 744 { 745 struct mt76_phy *main_phy = &phy->dev->phy; 746 int i; 747 748 mt76_txq_schedule_pending(phy); 749 750 if (phy != main_phy && phy->hw == main_phy->hw) 751 return; 752 753 for (i = 0; i <= MT_TXQ_BK; i++) 754 mt76_txq_schedule(phy, i); 755 } 756 EXPORT_SYMBOL_GPL(mt76_txq_schedule_all); 757 758 void mt76_tx_worker_run(struct mt76_dev *dev) 759 { 760 struct mt76_phy *phy; 761 int i; 762 763 mt76_txq_schedule_all(&dev->phy); 764 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 765 phy = dev->phys[i]; 766 if (!phy) 767 continue; 768 769 mt76_txq_schedule_all(phy); 770 } 771 772 #ifdef CONFIG_NL80211_TESTMODE 773 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 774 phy = dev->phys[i]; 775 if (!phy || !phy->test.tx_pending) 776 continue; 777 778 mt76_testmode_tx_pending(phy); 779 } 780 #endif 781 } 782 EXPORT_SYMBOL_GPL(mt76_tx_worker_run); 783 784 void mt76_tx_worker(struct mt76_worker *w) 785 { 786 struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker); 787 788 mt76_tx_worker_run(dev); 789 } 790 791 void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta, 792 bool send_bar) 793 { 794 int i; 795 796 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 797 struct ieee80211_txq *txq = sta->txq[i]; 798 struct mt76_queue *hwq; 799 struct mt76_txq *mtxq; 800 801 if (!txq) 802 continue; 803 804 hwq = phy->q_tx[mt76_txq_get_qid(txq)]; 805 mtxq = (struct mt76_txq *)txq->drv_priv; 806 807 spin_lock_bh(&hwq->lock); 808 mtxq->send_bar = mtxq->aggr && send_bar; 809 spin_unlock_bh(&hwq->lock); 810 } 811 } 812 EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); 813 814 void mt76_sta_ps_transition(struct mt76_dev *dev, struct mt76_wcid *wcid, 815 bool ps) 816 { 817 struct ieee80211_sta *sta; 818 struct ieee80211_hw *hw; 819 int i; 820 821 if (ps) { 822 set_bit(MT_WCID_FLAG_PS, &wcid->flags); 823 mt76_worker_schedule(&dev->tx_worker); 824 return; 825 } 826 827 clear_bit(MT_WCID_FLAG_PS, &wcid->flags); 828 829 sta = wcid_to_sta(wcid); 830 if (!sta) 831 return; 832 833 hw = mt76_phy_hw(dev, wcid->phy_idx); 834 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) 835 if (sta->txq[i]) 836 ieee80211_schedule_txq(hw, sta->txq[i]); 837 mt76_worker_schedule(&dev->tx_worker); 838 } 839 EXPORT_SYMBOL_GPL(mt76_sta_ps_transition); 840 841 void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) 842 { 843 struct mt76_phy *phy = hw->priv; 844 struct mt76_dev *dev = phy->dev; 845 846 mt76_worker_schedule(&dev->tx_worker); 847 } 848 EXPORT_SYMBOL_GPL(mt76_wake_tx_queue); 849 850 u8 mt76_ac_to_hwq(u8 ac) 851 { 852 static const u8 wmm_queue_map[] = { 853 [IEEE80211_AC_BE] = 0, 854 [IEEE80211_AC_BK] = 1, 855 [IEEE80211_AC_VI] = 2, 856 [IEEE80211_AC_VO] = 3, 857 }; 858 859 if (WARN_ON(ac >= IEEE80211_NUM_ACS)) 860 return 0; 861 862 return wmm_queue_map[ac]; 863 } 864 EXPORT_SYMBOL_GPL(mt76_ac_to_hwq); 865 866 int mt76_skb_adjust_pad(struct sk_buff *skb, int pad) 867 { 868 struct sk_buff *iter, *last = skb; 869 870 /* First packet of a A-MSDU burst keeps track of the whole burst 871 * length, need to update length of it and the last packet. 872 */ 873 skb_walk_frags(skb, iter) { 874 last = iter; 875 if (!iter->next) { 876 skb->data_len += pad; 877 skb->len += pad; 878 break; 879 } 880 } 881 882 if (skb_pad(last, pad)) 883 return -ENOMEM; 884 885 __skb_put(last, pad); 886 887 return 0; 888 } 889 EXPORT_SYMBOL_GPL(mt76_skb_adjust_pad); 890 891 void mt76_queue_tx_complete(struct mt76_dev *dev, struct mt76_queue *q, 892 struct mt76_queue_entry *e) 893 { 894 if (e->skb) 895 dev->drv->tx_complete_skb(dev, e); 896 897 spin_lock_bh(&q->lock); 898 q->tail = (q->tail + 1) % q->ndesc; 899 q->queued--; 900 spin_unlock_bh(&q->lock); 901 } 902 EXPORT_SYMBOL_GPL(mt76_queue_tx_complete); 903 904 void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked) 905 { 906 struct mt76_phy *phy = &dev->phy; 907 struct mt76_queue *q = phy->q_tx[0]; 908 909 if (blocked == q->blocked) 910 return; 911 912 q->blocked = blocked; 913 914 phy = dev->phys[MT_BAND1]; 915 if (phy) { 916 q = phy->q_tx[0]; 917 q->blocked = blocked; 918 } 919 phy = dev->phys[MT_BAND2]; 920 if (phy) { 921 q = phy->q_tx[0]; 922 q->blocked = blocked; 923 } 924 925 if (!blocked) 926 mt76_worker_schedule(&dev->tx_worker); 927 } 928 EXPORT_SYMBOL_GPL(__mt76_set_tx_blocked); 929 930 int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) 931 { 932 int token; 933 934 spin_lock_bh(&dev->token_lock); 935 936 token = idr_alloc(&dev->token, *ptxwi, dev->token_start, 937 dev->token_start + dev->token_size, 938 GFP_ATOMIC); 939 if (token >= dev->token_start) { 940 dev->token_count++; 941 942 if ((*ptxwi)->qid == MT_TXQ_PSD) { 943 struct mt76_phy *mphy = mt76_dev_phy(dev, (*ptxwi)->phy_idx); 944 atomic_inc(&mphy->mgmt_tx_pending); 945 } 946 } 947 948 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 949 if (mtk_wed_device_active(&dev->mmio.wed) && 950 token >= dev->mmio.wed.wlan.token_start) 951 dev->wed_token_count++; 952 #endif 953 954 if (dev->token_count >= dev->token_size - MT76_TOKEN_FREE_THR) 955 __mt76_set_tx_blocked(dev, true); 956 957 spin_unlock_bh(&dev->token_lock); 958 959 return token; 960 } 961 EXPORT_SYMBOL_GPL(mt76_token_consume); 962 963 int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr, 964 struct mt76_txwi_cache *t, dma_addr_t phys) 965 { 966 int token; 967 968 spin_lock_bh(&dev->rx_token_lock); 969 token = idr_alloc(&dev->rx_token, t, 0, dev->rx_token_size, 970 GFP_ATOMIC); 971 if (token >= 0) { 972 t->ptr = ptr; 973 t->dma_addr = phys; 974 } 975 spin_unlock_bh(&dev->rx_token_lock); 976 977 return token; 978 } 979 EXPORT_SYMBOL_GPL(mt76_rx_token_consume); 980 981 struct mt76_txwi_cache * 982 mt76_token_release(struct mt76_dev *dev, int token, bool *wake) 983 { 984 struct mt76_txwi_cache *txwi; 985 986 spin_lock_bh(&dev->token_lock); 987 988 txwi = idr_remove(&dev->token, token); 989 if (txwi) { 990 dev->token_count--; 991 992 if (txwi->qid == MT_TXQ_PSD) { 993 struct mt76_phy *mphy = mt76_dev_phy(dev, txwi->phy_idx); 994 if (atomic_dec_and_test(&mphy->mgmt_tx_pending)) 995 wake_up(&dev->tx_wait); 996 } 997 998 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 999 if (mtk_wed_device_active(&dev->mmio.wed) && 1000 token >= dev->mmio.wed.wlan.token_start && 1001 --dev->wed_token_count == 0) 1002 wake_up(&dev->tx_wait); 1003 #endif 1004 } 1005 1006 if (wake && dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && 1007 dev->phy.q_tx[0]->blocked) 1008 *wake = true; 1009 1010 spin_unlock_bh(&dev->token_lock); 1011 1012 return txwi; 1013 } 1014 EXPORT_SYMBOL_GPL(mt76_token_release); 1015 1016 struct mt76_txwi_cache * 1017 mt76_rx_token_release(struct mt76_dev *dev, int token) 1018 { 1019 struct mt76_txwi_cache *t; 1020 1021 spin_lock_bh(&dev->rx_token_lock); 1022 t = idr_remove(&dev->rx_token, token); 1023 spin_unlock_bh(&dev->rx_token_lock); 1024 1025 return t; 1026 } 1027 EXPORT_SYMBOL_GPL(mt76_rx_token_release); 1028