1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include "mt76.h" 7 8 static int 9 mt76_txq_get_qid(struct ieee80211_txq *txq) 10 { 11 if (!txq->sta) 12 return MT_TXQ_BE; 13 14 return txq->ac; 15 } 16 17 void 18 mt76_tx_check_agg_ssn(struct ieee80211_sta *sta, struct sk_buff *skb) 19 { 20 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 21 struct ieee80211_txq *txq; 22 struct mt76_txq *mtxq; 23 u8 tid; 24 25 if (!sta || !ieee80211_is_data_qos(hdr->frame_control) || 26 !ieee80211_is_data_present(hdr->frame_control)) 27 return; 28 29 tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; 30 txq = sta->txq[tid]; 31 mtxq = (struct mt76_txq *)txq->drv_priv; 32 if (!mtxq->aggr) 33 return; 34 35 mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10; 36 } 37 EXPORT_SYMBOL_GPL(mt76_tx_check_agg_ssn); 38 39 void 40 mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list) 41 __acquires(&dev->status_lock) 42 { 43 __skb_queue_head_init(list); 44 spin_lock_bh(&dev->status_lock); 45 } 46 EXPORT_SYMBOL_GPL(mt76_tx_status_lock); 47 48 void 49 mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) 50 __releases(&dev->status_lock) 51 { 52 struct ieee80211_hw *hw; 53 struct sk_buff *skb; 54 55 spin_unlock_bh(&dev->status_lock); 56 57 rcu_read_lock(); 58 while ((skb = __skb_dequeue(list)) != NULL) { 59 struct ieee80211_tx_status status = { 60 .skb = skb, 61 .info = IEEE80211_SKB_CB(skb), 62 }; 63 struct ieee80211_rate_status rs = {}; 64 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 65 struct mt76_wcid *wcid; 66 67 wcid = __mt76_wcid_ptr(dev, cb->wcid); 68 if (wcid) { 69 status.sta = wcid_to_sta(wcid); 70 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 71 rs.rate_idx = wcid->rate; 72 status.rates = &rs; 73 status.n_rates = 1; 74 } else { 75 status.n_rates = 0; 76 } 77 } 78 79 hw = mt76_tx_status_get_hw(dev, skb); 80 spin_lock_bh(&dev->rx_lock); 81 ieee80211_tx_status_ext(hw, &status); 82 spin_unlock_bh(&dev->rx_lock); 83 } 84 rcu_read_unlock(); 85 } 86 EXPORT_SYMBOL_GPL(mt76_tx_status_unlock); 87 88 static void 89 __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, 90 struct sk_buff_head *list) 91 { 92 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 93 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 94 u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE; 95 96 flags |= cb->flags; 97 cb->flags = flags; 98 99 if ((flags & done) != done) 100 return; 101 102 /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ 103 if (flags & MT_TX_CB_TXS_FAILED && 104 (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) { 105 info->status.rates[0].count = 0; 106 info->status.rates[0].idx = -1; 107 info->flags |= IEEE80211_TX_STAT_ACK; 108 } 109 110 __skb_queue_tail(list, skb); 111 } 112 113 void 114 mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, 115 struct sk_buff_head *list) 116 { 117 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list); 118 } 119 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done); 120 121 int 122 mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid, 123 struct sk_buff *skb) 124 { 125 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 126 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 127 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 128 int pid; 129 130 memset(cb, 0, sizeof(*cb)); 131 132 if (!wcid || !rcu_access_pointer(dev->wcid[wcid->idx])) 133 return MT_PACKET_ID_NO_ACK; 134 135 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 136 return MT_PACKET_ID_NO_ACK; 137 138 if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | 139 IEEE80211_TX_CTL_RATE_CTRL_PROBE))) { 140 if (mtk_wed_device_active(&dev->mmio.wed) && 141 ((info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) || 142 ieee80211_is_data(hdr->frame_control))) 143 return MT_PACKET_ID_WED; 144 145 return MT_PACKET_ID_NO_SKB; 146 } 147 148 spin_lock_bh(&dev->status_lock); 149 150 pid = idr_alloc(&wcid->pktid, skb, MT_PACKET_ID_FIRST, 151 MT_PACKET_ID_MASK, GFP_ATOMIC); 152 if (pid < 0) { 153 pid = MT_PACKET_ID_NO_SKB; 154 goto out; 155 } 156 157 cb->wcid = wcid->idx; 158 cb->pktid = pid; 159 160 if (list_empty(&wcid->list)) 161 list_add_tail(&wcid->list, &dev->wcid_list); 162 163 out: 164 spin_unlock_bh(&dev->status_lock); 165 166 return pid; 167 } 168 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add); 169 170 struct sk_buff * 171 mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid, 172 struct sk_buff_head *list) 173 { 174 struct sk_buff *skb; 175 int id; 176 177 lockdep_assert_held(&dev->status_lock); 178 179 skb = idr_remove(&wcid->pktid, pktid); 180 if (skb) 181 goto out; 182 183 /* look for stale entries in the wcid idr queue */ 184 idr_for_each_entry(&wcid->pktid, skb, id) { 185 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 186 187 if (pktid >= 0) { 188 if (!(cb->flags & MT_TX_CB_DMA_DONE)) 189 continue; 190 191 if (time_is_after_jiffies(cb->jiffies + 192 MT_TX_STATUS_SKB_TIMEOUT)) 193 continue; 194 } 195 196 /* It has been too long since DMA_DONE, time out this packet 197 * and stop waiting for TXS callback. 198 */ 199 idr_remove(&wcid->pktid, cb->pktid); 200 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED | 201 MT_TX_CB_TXS_DONE, list); 202 } 203 204 out: 205 if (idr_is_empty(&wcid->pktid)) 206 list_del_init(&wcid->list); 207 208 return skb; 209 } 210 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get); 211 212 void 213 mt76_tx_status_check(struct mt76_dev *dev, bool flush) 214 { 215 struct mt76_wcid *wcid, *tmp; 216 struct sk_buff_head list; 217 218 mt76_tx_status_lock(dev, &list); 219 list_for_each_entry_safe(wcid, tmp, &dev->wcid_list, list) 220 mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list); 221 mt76_tx_status_unlock(dev, &list); 222 } 223 EXPORT_SYMBOL_GPL(mt76_tx_status_check); 224 225 static void 226 mt76_tx_check_non_aql(struct mt76_dev *dev, struct mt76_wcid *wcid, 227 struct sk_buff *skb) 228 { 229 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 230 struct ieee80211_sta *sta; 231 int pending; 232 int i; 233 234 if (!wcid || info->tx_time_est) 235 return; 236 237 pending = atomic_dec_return(&wcid->non_aql_packets); 238 if (pending < 0) 239 atomic_cmpxchg(&wcid->non_aql_packets, pending, 0); 240 241 sta = wcid_to_sta(wcid); 242 if (!sta || pending != MT_MAX_NON_AQL_PKT - 1) 243 return; 244 245 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 246 if (!sta->txq[i]) 247 continue; 248 249 ieee80211_schedule_txq(dev->hw, sta->txq[i]); 250 } 251 } 252 253 void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb, 254 struct list_head *free_list) 255 { 256 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 257 struct ieee80211_tx_status status = { 258 .skb = skb, 259 .free_list = free_list, 260 }; 261 struct mt76_wcid *wcid = NULL; 262 struct ieee80211_hw *hw; 263 struct sk_buff_head list; 264 265 rcu_read_lock(); 266 267 wcid = __mt76_wcid_ptr(dev, wcid_idx); 268 mt76_tx_check_non_aql(dev, wcid, skb); 269 270 #ifdef CONFIG_NL80211_TESTMODE 271 if (mt76_is_testmode_skb(dev, skb, &hw)) { 272 struct mt76_phy *phy = hw->priv; 273 274 if (skb == phy->test.tx_skb) 275 phy->test.tx_done++; 276 if (phy->test.tx_queued == phy->test.tx_done) 277 wake_up(&dev->tx_wait); 278 279 dev_kfree_skb_any(skb); 280 goto out; 281 } 282 #endif 283 284 if (cb->pktid < MT_PACKET_ID_FIRST) { 285 struct ieee80211_rate_status rs = {}; 286 287 hw = mt76_tx_status_get_hw(dev, skb); 288 status.sta = wcid_to_sta(wcid); 289 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 290 rs.rate_idx = wcid->rate; 291 status.rates = &rs; 292 status.n_rates = 1; 293 } 294 spin_lock_bh(&dev->rx_lock); 295 ieee80211_tx_status_ext(hw, &status); 296 spin_unlock_bh(&dev->rx_lock); 297 goto out; 298 } 299 300 mt76_tx_status_lock(dev, &list); 301 cb->jiffies = jiffies; 302 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list); 303 mt76_tx_status_unlock(dev, &list); 304 305 out: 306 rcu_read_unlock(); 307 } 308 EXPORT_SYMBOL_GPL(__mt76_tx_complete_skb); 309 310 static int 311 __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, 312 struct mt76_wcid *wcid, struct ieee80211_sta *sta, 313 bool *stop) 314 { 315 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 316 struct mt76_queue *q = phy->q_tx[qid]; 317 struct mt76_dev *dev = phy->dev; 318 bool non_aql; 319 int pending; 320 int idx; 321 322 non_aql = !info->tx_time_est; 323 idx = dev->queue_ops->tx_queue_skb(phy, q, qid, skb, wcid, sta); 324 if (idx < 0 || !sta) 325 return idx; 326 327 wcid = (struct mt76_wcid *)sta->drv_priv; 328 if (!wcid->sta) 329 return idx; 330 331 q->entry[idx].wcid = wcid->idx; 332 333 if (!non_aql) 334 return idx; 335 336 pending = atomic_inc_return(&wcid->non_aql_packets); 337 if (stop && pending >= MT_MAX_NON_AQL_PKT) 338 *stop = true; 339 340 return idx; 341 } 342 343 void 344 mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta, 345 struct mt76_wcid *wcid, struct sk_buff *skb) 346 { 347 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 348 struct ieee80211_hdr *hdr = (void *)skb->data; 349 struct sk_buff_head *head; 350 351 if (mt76_testmode_enabled(phy)) { 352 ieee80211_free_txskb(phy->hw, skb); 353 return; 354 } 355 356 if (WARN_ON(skb_get_queue_mapping(skb) >= MT_TXQ_PSD)) 357 skb_set_queue_mapping(skb, MT_TXQ_BE); 358 359 if (wcid && !(wcid->tx_info & MT_WCID_TX_INFO_SET)) 360 ieee80211_get_tx_rates(info->control.vif, sta, skb, 361 info->control.rates, 1); 362 363 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 364 365 if ((info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) || 366 ((info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK) && 367 ieee80211_is_probe_req(hdr->frame_control))) 368 head = &wcid->tx_offchannel; 369 else 370 head = &wcid->tx_pending; 371 372 spin_lock_bh(&head->lock); 373 __skb_queue_tail(head, skb); 374 spin_unlock_bh(&head->lock); 375 376 spin_lock_bh(&phy->tx_lock); 377 if (list_empty(&wcid->tx_list)) 378 list_add_tail(&wcid->tx_list, &phy->tx_list); 379 spin_unlock_bh(&phy->tx_lock); 380 381 mt76_worker_schedule(&phy->dev->tx_worker); 382 } 383 EXPORT_SYMBOL_GPL(mt76_tx); 384 385 static struct sk_buff * 386 mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq) 387 { 388 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 389 struct ieee80211_tx_info *info; 390 struct sk_buff *skb; 391 392 skb = ieee80211_tx_dequeue(phy->hw, txq); 393 if (!skb) 394 return NULL; 395 396 info = IEEE80211_SKB_CB(skb); 397 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 398 399 return skb; 400 } 401 402 static void 403 mt76_queue_ps_skb(struct mt76_phy *phy, struct ieee80211_sta *sta, 404 struct sk_buff *skb, bool last) 405 { 406 struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; 407 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 408 409 info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE; 410 if (last) 411 info->flags |= IEEE80211_TX_STATUS_EOSP | 412 IEEE80211_TX_CTL_REQ_TX_STATUS; 413 414 mt76_skb_set_moredata(skb, !last); 415 __mt76_tx_queue_skb(phy, MT_TXQ_PSD, skb, wcid, sta, NULL); 416 } 417 418 void 419 mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, 420 u16 tids, int nframes, 421 enum ieee80211_frame_release_type reason, 422 bool more_data) 423 { 424 struct mt76_phy *phy = hw->priv; 425 struct mt76_dev *dev = phy->dev; 426 struct sk_buff *last_skb = NULL; 427 struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD]; 428 int i; 429 430 spin_lock_bh(&hwq->lock); 431 for (i = 0; tids && nframes; i++, tids >>= 1) { 432 struct ieee80211_txq *txq = sta->txq[i]; 433 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 434 struct sk_buff *skb; 435 436 if (!(tids & 1)) 437 continue; 438 439 do { 440 skb = mt76_txq_dequeue(phy, mtxq); 441 if (!skb) 442 break; 443 444 nframes--; 445 if (last_skb) 446 mt76_queue_ps_skb(phy, sta, last_skb, false); 447 448 last_skb = skb; 449 } while (nframes); 450 } 451 452 if (last_skb) { 453 mt76_queue_ps_skb(phy, sta, last_skb, true); 454 dev->queue_ops->kick(dev, hwq); 455 } else { 456 ieee80211_sta_eosp(sta); 457 } 458 459 spin_unlock_bh(&hwq->lock); 460 } 461 EXPORT_SYMBOL_GPL(mt76_release_buffered_frames); 462 463 static bool 464 mt76_txq_stopped(struct mt76_queue *q) 465 { 466 return q->stopped || q->blocked || 467 q->queued + MT_TXQ_FREE_THR >= q->ndesc; 468 } 469 470 static int 471 mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, 472 struct mt76_txq *mtxq, struct mt76_wcid *wcid) 473 { 474 struct mt76_dev *dev = phy->dev; 475 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 476 enum mt76_txq_id qid = mt76_txq_get_qid(txq); 477 struct ieee80211_tx_info *info; 478 struct sk_buff *skb; 479 int n_frames = 1; 480 bool stop = false; 481 int idx; 482 483 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 484 return 0; 485 486 if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) 487 return 0; 488 489 skb = mt76_txq_dequeue(phy, mtxq); 490 if (!skb) 491 return 0; 492 493 info = IEEE80211_SKB_CB(skb); 494 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 495 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 496 info->control.rates, 1); 497 498 spin_lock(&q->lock); 499 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 500 spin_unlock(&q->lock); 501 if (idx < 0) 502 return idx; 503 504 do { 505 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 506 break; 507 508 if (stop || mt76_txq_stopped(q)) 509 break; 510 511 skb = mt76_txq_dequeue(phy, mtxq); 512 if (!skb) 513 break; 514 515 info = IEEE80211_SKB_CB(skb); 516 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 517 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 518 info->control.rates, 1); 519 520 spin_lock(&q->lock); 521 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 522 spin_unlock(&q->lock); 523 if (idx < 0) 524 break; 525 526 n_frames++; 527 } while (1); 528 529 spin_lock(&q->lock); 530 dev->queue_ops->kick(dev, q); 531 spin_unlock(&q->lock); 532 533 return n_frames; 534 } 535 536 static int 537 mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) 538 { 539 struct mt76_dev *dev = phy->dev; 540 struct ieee80211_txq *txq; 541 struct mt76_txq *mtxq; 542 struct mt76_wcid *wcid; 543 struct mt76_queue *q; 544 int ret = 0; 545 546 while (1) { 547 int n_frames = 0; 548 549 txq = ieee80211_next_txq(phy->hw, qid); 550 if (!txq) 551 break; 552 553 mtxq = (struct mt76_txq *)txq->drv_priv; 554 wcid = __mt76_wcid_ptr(dev, mtxq->wcid); 555 if (!wcid || test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 556 continue; 557 558 if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) 559 continue; 560 561 phy = mt76_dev_phy(dev, wcid->phy_idx); 562 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 563 continue; 564 565 q = phy->q_tx[qid]; 566 if (dev->queue_ops->tx_cleanup && 567 q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { 568 dev->queue_ops->tx_cleanup(dev, q, false); 569 } 570 571 if (mtxq->send_bar && mtxq->aggr) { 572 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 573 struct ieee80211_sta *sta = txq->sta; 574 struct ieee80211_vif *vif = txq->vif; 575 u16 agg_ssn = mtxq->agg_ssn; 576 u8 tid = txq->tid; 577 578 mtxq->send_bar = false; 579 ieee80211_send_bar(vif, sta->addr, tid, agg_ssn); 580 } 581 582 if (!mt76_txq_stopped(q)) 583 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 584 585 ieee80211_return_txq(phy->hw, txq, false); 586 587 if (unlikely(n_frames < 0)) 588 return n_frames; 589 590 ret += n_frames; 591 } 592 593 return ret; 594 } 595 596 void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid) 597 { 598 int len; 599 600 if (qid >= 4) 601 return; 602 603 local_bh_disable(); 604 rcu_read_lock(); 605 606 do { 607 ieee80211_txq_schedule_start(phy->hw, qid); 608 len = mt76_txq_schedule_list(phy, qid); 609 ieee80211_txq_schedule_end(phy->hw, qid); 610 } while (len > 0); 611 612 rcu_read_unlock(); 613 local_bh_enable(); 614 } 615 EXPORT_SYMBOL_GPL(mt76_txq_schedule); 616 617 static int 618 mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, 619 struct sk_buff_head *head) 620 { 621 struct mt76_dev *dev = phy->dev; 622 struct ieee80211_sta *sta; 623 struct mt76_queue *q; 624 struct sk_buff *skb; 625 int ret = 0; 626 627 spin_lock(&head->lock); 628 while ((skb = skb_peek(head)) != NULL) { 629 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 630 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 631 int qid = skb_get_queue_mapping(skb); 632 633 if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) && 634 !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && 635 !ieee80211_is_data_present(hdr->frame_control) && 636 (!ieee80211_is_bufferable_mmpdu(skb) || 637 ieee80211_is_deauth(hdr->frame_control) || 638 head == &wcid->tx_offchannel)) 639 qid = MT_TXQ_PSD; 640 641 q = phy->q_tx[qid]; 642 if (mt76_txq_stopped(q) || test_bit(MT76_RESET, &phy->state)) { 643 ret = -1; 644 break; 645 } 646 647 __skb_unlink(skb, head); 648 spin_unlock(&head->lock); 649 650 sta = wcid_to_sta(wcid); 651 spin_lock(&q->lock); 652 __mt76_tx_queue_skb(phy, qid, skb, wcid, sta, NULL); 653 dev->queue_ops->kick(dev, q); 654 spin_unlock(&q->lock); 655 656 spin_lock(&head->lock); 657 } 658 spin_unlock(&head->lock); 659 660 return ret; 661 } 662 663 void mt76_txq_schedule_pending(struct mt76_phy *phy) 664 { 665 LIST_HEAD(tx_list); 666 int ret = 0; 667 668 if (list_empty(&phy->tx_list)) 669 return; 670 671 local_bh_disable(); 672 rcu_read_lock(); 673 674 spin_lock(&phy->tx_lock); 675 list_splice_init(&phy->tx_list, &tx_list); 676 while (!list_empty(&tx_list)) { 677 struct mt76_wcid *wcid; 678 679 wcid = list_first_entry(&tx_list, struct mt76_wcid, tx_list); 680 list_del_init(&wcid->tx_list); 681 682 spin_unlock(&phy->tx_lock); 683 if (ret >= 0) 684 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel); 685 if (ret >= 0 && !phy->offchannel) 686 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); 687 spin_lock(&phy->tx_lock); 688 689 if (!skb_queue_empty(&wcid->tx_pending) && 690 !skb_queue_empty(&wcid->tx_offchannel) && 691 list_empty(&wcid->tx_list)) 692 list_add_tail(&wcid->tx_list, &phy->tx_list); 693 } 694 spin_unlock(&phy->tx_lock); 695 696 rcu_read_unlock(); 697 local_bh_enable(); 698 } 699 700 void mt76_txq_schedule_all(struct mt76_phy *phy) 701 { 702 struct mt76_phy *main_phy = &phy->dev->phy; 703 int i; 704 705 mt76_txq_schedule_pending(phy); 706 707 if (phy != main_phy && phy->hw == main_phy->hw) 708 return; 709 710 for (i = 0; i <= MT_TXQ_BK; i++) 711 mt76_txq_schedule(phy, i); 712 } 713 EXPORT_SYMBOL_GPL(mt76_txq_schedule_all); 714 715 void mt76_tx_worker_run(struct mt76_dev *dev) 716 { 717 struct mt76_phy *phy; 718 int i; 719 720 mt76_txq_schedule_all(&dev->phy); 721 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 722 phy = dev->phys[i]; 723 if (!phy) 724 continue; 725 726 mt76_txq_schedule_all(phy); 727 } 728 729 #ifdef CONFIG_NL80211_TESTMODE 730 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 731 phy = dev->phys[i]; 732 if (!phy || !phy->test.tx_pending) 733 continue; 734 735 mt76_testmode_tx_pending(phy); 736 } 737 #endif 738 } 739 EXPORT_SYMBOL_GPL(mt76_tx_worker_run); 740 741 void mt76_tx_worker(struct mt76_worker *w) 742 { 743 struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker); 744 745 mt76_tx_worker_run(dev); 746 } 747 748 void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta, 749 bool send_bar) 750 { 751 int i; 752 753 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 754 struct ieee80211_txq *txq = sta->txq[i]; 755 struct mt76_queue *hwq; 756 struct mt76_txq *mtxq; 757 758 if (!txq) 759 continue; 760 761 hwq = phy->q_tx[mt76_txq_get_qid(txq)]; 762 mtxq = (struct mt76_txq *)txq->drv_priv; 763 764 spin_lock_bh(&hwq->lock); 765 mtxq->send_bar = mtxq->aggr && send_bar; 766 spin_unlock_bh(&hwq->lock); 767 } 768 } 769 EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); 770 771 void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) 772 { 773 struct mt76_phy *phy = hw->priv; 774 struct mt76_dev *dev = phy->dev; 775 776 mt76_worker_schedule(&dev->tx_worker); 777 } 778 EXPORT_SYMBOL_GPL(mt76_wake_tx_queue); 779 780 u8 mt76_ac_to_hwq(u8 ac) 781 { 782 static const u8 wmm_queue_map[] = { 783 [IEEE80211_AC_BE] = 0, 784 [IEEE80211_AC_BK] = 1, 785 [IEEE80211_AC_VI] = 2, 786 [IEEE80211_AC_VO] = 3, 787 }; 788 789 if (WARN_ON(ac >= IEEE80211_NUM_ACS)) 790 return 0; 791 792 return wmm_queue_map[ac]; 793 } 794 EXPORT_SYMBOL_GPL(mt76_ac_to_hwq); 795 796 int mt76_skb_adjust_pad(struct sk_buff *skb, int pad) 797 { 798 struct sk_buff *iter, *last = skb; 799 800 /* First packet of a A-MSDU burst keeps track of the whole burst 801 * length, need to update length of it and the last packet. 802 */ 803 skb_walk_frags(skb, iter) { 804 last = iter; 805 if (!iter->next) { 806 skb->data_len += pad; 807 skb->len += pad; 808 break; 809 } 810 } 811 812 if (skb_pad(last, pad)) 813 return -ENOMEM; 814 815 __skb_put(last, pad); 816 817 return 0; 818 } 819 EXPORT_SYMBOL_GPL(mt76_skb_adjust_pad); 820 821 void mt76_queue_tx_complete(struct mt76_dev *dev, struct mt76_queue *q, 822 struct mt76_queue_entry *e) 823 { 824 if (e->skb) 825 dev->drv->tx_complete_skb(dev, e); 826 827 spin_lock_bh(&q->lock); 828 q->tail = (q->tail + 1) % q->ndesc; 829 q->queued--; 830 spin_unlock_bh(&q->lock); 831 } 832 EXPORT_SYMBOL_GPL(mt76_queue_tx_complete); 833 834 void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked) 835 { 836 struct mt76_phy *phy = &dev->phy; 837 struct mt76_queue *q = phy->q_tx[0]; 838 839 if (blocked == q->blocked) 840 return; 841 842 q->blocked = blocked; 843 844 phy = dev->phys[MT_BAND1]; 845 if (phy) { 846 q = phy->q_tx[0]; 847 q->blocked = blocked; 848 } 849 phy = dev->phys[MT_BAND2]; 850 if (phy) { 851 q = phy->q_tx[0]; 852 q->blocked = blocked; 853 } 854 855 if (!blocked) 856 mt76_worker_schedule(&dev->tx_worker); 857 } 858 EXPORT_SYMBOL_GPL(__mt76_set_tx_blocked); 859 860 int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) 861 { 862 int token; 863 864 spin_lock_bh(&dev->token_lock); 865 866 token = idr_alloc(&dev->token, *ptxwi, dev->token_start, 867 dev->token_start + dev->token_size, 868 GFP_ATOMIC); 869 if (token >= dev->token_start) { 870 dev->token_count++; 871 872 if ((*ptxwi)->qid == MT_TXQ_PSD) { 873 struct mt76_phy *mphy = mt76_dev_phy(dev, (*ptxwi)->phy_idx); 874 atomic_inc(&mphy->mgmt_tx_pending); 875 } 876 } 877 878 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 879 if (mtk_wed_device_active(&dev->mmio.wed) && 880 token >= dev->mmio.wed.wlan.token_start) 881 dev->wed_token_count++; 882 #endif 883 884 if (dev->token_count >= dev->token_size - MT76_TOKEN_FREE_THR) 885 __mt76_set_tx_blocked(dev, true); 886 887 spin_unlock_bh(&dev->token_lock); 888 889 return token; 890 } 891 EXPORT_SYMBOL_GPL(mt76_token_consume); 892 893 int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr, 894 struct mt76_txwi_cache *t, dma_addr_t phys) 895 { 896 int token; 897 898 spin_lock_bh(&dev->rx_token_lock); 899 token = idr_alloc(&dev->rx_token, t, 0, dev->rx_token_size, 900 GFP_ATOMIC); 901 if (token >= 0) { 902 t->ptr = ptr; 903 t->dma_addr = phys; 904 } 905 spin_unlock_bh(&dev->rx_token_lock); 906 907 return token; 908 } 909 EXPORT_SYMBOL_GPL(mt76_rx_token_consume); 910 911 struct mt76_txwi_cache * 912 mt76_token_release(struct mt76_dev *dev, int token, bool *wake) 913 { 914 struct mt76_txwi_cache *txwi; 915 916 spin_lock_bh(&dev->token_lock); 917 918 txwi = idr_remove(&dev->token, token); 919 if (txwi) { 920 dev->token_count--; 921 922 if (txwi->qid == MT_TXQ_PSD) { 923 struct mt76_phy *mphy = mt76_dev_phy(dev, txwi->phy_idx); 924 if (atomic_dec_and_test(&mphy->mgmt_tx_pending)) 925 wake_up(&dev->tx_wait); 926 } 927 928 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 929 if (mtk_wed_device_active(&dev->mmio.wed) && 930 token >= dev->mmio.wed.wlan.token_start && 931 --dev->wed_token_count == 0) 932 wake_up(&dev->tx_wait); 933 #endif 934 } 935 936 if (dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && 937 dev->phy.q_tx[0]->blocked) 938 *wake = true; 939 940 spin_unlock_bh(&dev->token_lock); 941 942 return txwi; 943 } 944 EXPORT_SYMBOL_GPL(mt76_token_release); 945 946 struct mt76_txwi_cache * 947 mt76_rx_token_release(struct mt76_dev *dev, int token) 948 { 949 struct mt76_txwi_cache *t; 950 951 spin_lock_bh(&dev->rx_token_lock); 952 t = idr_remove(&dev->rx_token, token); 953 spin_unlock_bh(&dev->rx_token_lock); 954 955 return t; 956 } 957 EXPORT_SYMBOL_GPL(mt76_rx_token_release); 958