1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include "mt76.h" 7 8 static int 9 mt76_txq_get_qid(struct ieee80211_txq *txq) 10 { 11 if (!txq->sta) 12 return MT_TXQ_BE; 13 14 return txq->ac; 15 } 16 17 void 18 mt76_tx_check_agg_ssn(struct ieee80211_sta *sta, struct sk_buff *skb) 19 { 20 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 21 struct ieee80211_txq *txq; 22 struct mt76_txq *mtxq; 23 u8 tid; 24 25 if (!sta || !ieee80211_is_data_qos(hdr->frame_control) || 26 !ieee80211_is_data_present(hdr->frame_control)) 27 return; 28 29 tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; 30 txq = sta->txq[tid]; 31 mtxq = (struct mt76_txq *)txq->drv_priv; 32 if (!mtxq->aggr) 33 return; 34 35 mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10; 36 } 37 EXPORT_SYMBOL_GPL(mt76_tx_check_agg_ssn); 38 39 void 40 mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list) 41 __acquires(&dev->status_lock) 42 { 43 __skb_queue_head_init(list); 44 spin_lock_bh(&dev->status_lock); 45 } 46 EXPORT_SYMBOL_GPL(mt76_tx_status_lock); 47 48 void 49 mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) 50 __releases(&dev->status_lock) 51 { 52 struct ieee80211_hw *hw; 53 struct sk_buff *skb; 54 55 spin_unlock_bh(&dev->status_lock); 56 57 rcu_read_lock(); 58 while ((skb = __skb_dequeue(list)) != NULL) { 59 struct ieee80211_tx_status status = { 60 .skb = skb, 61 .info = IEEE80211_SKB_CB(skb), 62 }; 63 struct ieee80211_rate_status rs = {}; 64 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 65 struct mt76_wcid *wcid; 66 67 wcid = __mt76_wcid_ptr(dev, cb->wcid); 68 if (wcid) { 69 status.sta = wcid_to_sta(wcid); 70 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 71 rs.rate_idx = wcid->rate; 72 status.rates = &rs; 73 status.n_rates = 1; 74 } else { 75 status.n_rates = 0; 76 } 77 } 78 79 hw = mt76_tx_status_get_hw(dev, skb); 80 spin_lock_bh(&dev->rx_lock); 81 ieee80211_tx_status_ext(hw, &status); 82 spin_unlock_bh(&dev->rx_lock); 83 } 84 rcu_read_unlock(); 85 } 86 EXPORT_SYMBOL_GPL(mt76_tx_status_unlock); 87 88 static void 89 __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, 90 struct sk_buff_head *list) 91 { 92 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 93 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 94 u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE; 95 96 flags |= cb->flags; 97 cb->flags = flags; 98 99 if ((flags & done) != done) 100 return; 101 102 /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ 103 if (flags & MT_TX_CB_TXS_FAILED && 104 (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) { 105 info->status.rates[0].count = 0; 106 info->status.rates[0].idx = -1; 107 info->flags |= IEEE80211_TX_STAT_ACK; 108 } 109 110 __skb_queue_tail(list, skb); 111 } 112 113 void 114 mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, 115 struct sk_buff_head *list) 116 { 117 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list); 118 } 119 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done); 120 121 int 122 mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid, 123 struct sk_buff *skb) 124 { 125 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 126 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 127 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 128 int pid; 129 130 memset(cb, 0, sizeof(*cb)); 131 132 if (!wcid || !rcu_access_pointer(dev->wcid[wcid->idx])) 133 return MT_PACKET_ID_NO_ACK; 134 135 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 136 return MT_PACKET_ID_NO_ACK; 137 138 if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | 139 IEEE80211_TX_CTL_RATE_CTRL_PROBE))) { 140 if (mtk_wed_device_active(&dev->mmio.wed) && 141 ((info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) || 142 ieee80211_is_data(hdr->frame_control))) 143 return MT_PACKET_ID_WED; 144 145 return MT_PACKET_ID_NO_SKB; 146 } 147 148 spin_lock_bh(&dev->status_lock); 149 150 pid = idr_alloc(&wcid->pktid, skb, MT_PACKET_ID_FIRST, 151 MT_PACKET_ID_MASK, GFP_ATOMIC); 152 if (pid < 0) { 153 pid = MT_PACKET_ID_NO_SKB; 154 goto out; 155 } 156 157 cb->wcid = wcid->idx; 158 cb->pktid = pid; 159 160 if (list_empty(&wcid->list)) 161 list_add_tail(&wcid->list, &dev->wcid_list); 162 163 out: 164 spin_unlock_bh(&dev->status_lock); 165 166 return pid; 167 } 168 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add); 169 170 struct sk_buff * 171 mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid, 172 struct sk_buff_head *list) 173 { 174 struct sk_buff *skb; 175 int id; 176 177 lockdep_assert_held(&dev->status_lock); 178 179 skb = idr_remove(&wcid->pktid, pktid); 180 if (skb) 181 goto out; 182 183 /* look for stale entries in the wcid idr queue */ 184 idr_for_each_entry(&wcid->pktid, skb, id) { 185 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 186 187 if (pktid >= 0) { 188 if (!(cb->flags & MT_TX_CB_DMA_DONE)) 189 continue; 190 191 if (time_is_after_jiffies(cb->jiffies + 192 MT_TX_STATUS_SKB_TIMEOUT)) 193 continue; 194 } 195 196 /* It has been too long since DMA_DONE, time out this packet 197 * and stop waiting for TXS callback. 198 */ 199 idr_remove(&wcid->pktid, cb->pktid); 200 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED | 201 MT_TX_CB_TXS_DONE, list); 202 } 203 204 out: 205 if (idr_is_empty(&wcid->pktid)) 206 list_del_init(&wcid->list); 207 208 return skb; 209 } 210 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get); 211 212 void 213 mt76_tx_status_check(struct mt76_dev *dev, bool flush) 214 { 215 struct mt76_wcid *wcid, *tmp; 216 struct sk_buff_head list; 217 218 mt76_tx_status_lock(dev, &list); 219 list_for_each_entry_safe(wcid, tmp, &dev->wcid_list, list) 220 mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list); 221 mt76_tx_status_unlock(dev, &list); 222 } 223 EXPORT_SYMBOL_GPL(mt76_tx_status_check); 224 225 static void 226 mt76_tx_check_non_aql(struct mt76_dev *dev, struct mt76_wcid *wcid, 227 struct sk_buff *skb) 228 { 229 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 230 struct ieee80211_sta *sta; 231 int pending; 232 int i; 233 234 if (!wcid || info->tx_time_est) 235 return; 236 237 wcid = mt76_wcid_primary(wcid); 238 239 pending = atomic_dec_return(&wcid->non_aql_packets); 240 if (pending < 0) 241 atomic_cmpxchg(&wcid->non_aql_packets, pending, 0); 242 243 sta = wcid_to_sta(wcid); 244 if (!sta || pending != MT_MAX_NON_AQL_PKT - 1) 245 return; 246 247 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 248 if (!sta->txq[i]) 249 continue; 250 251 ieee80211_schedule_txq(dev->hw, sta->txq[i]); 252 } 253 } 254 255 void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb, 256 struct list_head *free_list) 257 { 258 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 259 struct ieee80211_tx_status status = { 260 .skb = skb, 261 .free_list = free_list, 262 }; 263 struct mt76_wcid *wcid = NULL; 264 struct ieee80211_hw *hw; 265 struct sk_buff_head list; 266 267 rcu_read_lock(); 268 269 wcid = __mt76_wcid_ptr(dev, wcid_idx); 270 mt76_tx_check_non_aql(dev, wcid, skb); 271 272 if (wcid && (dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING) && 273 test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 274 struct ieee80211_sta *sta = wcid_to_sta(wcid); 275 276 if (sta) { 277 struct ieee80211_hw *hw = mt76_phy_hw(dev, wcid->phy_idx); 278 int i; 279 280 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) 281 if (sta->txq[i]) 282 ieee80211_schedule_txq(hw, sta->txq[i]); 283 mt76_worker_schedule(&dev->tx_worker); 284 } 285 } 286 287 #ifdef CONFIG_NL80211_TESTMODE 288 if (mt76_is_testmode_skb(dev, skb, &hw)) { 289 struct mt76_phy *phy = hw->priv; 290 291 if (skb == phy->test.tx_skb) 292 phy->test.tx_done++; 293 if (phy->test.tx_queued == phy->test.tx_done) 294 wake_up(&dev->tx_wait); 295 296 dev_kfree_skb_any(skb); 297 goto out; 298 } 299 #endif 300 301 if (cb->pktid < MT_PACKET_ID_FIRST) { 302 struct ieee80211_rate_status rs = {}; 303 304 hw = mt76_tx_status_get_hw(dev, skb); 305 status.sta = wcid_to_sta(wcid); 306 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 307 rs.rate_idx = wcid->rate; 308 status.rates = &rs; 309 status.n_rates = 1; 310 } 311 spin_lock_bh(&dev->rx_lock); 312 ieee80211_tx_status_ext(hw, &status); 313 spin_unlock_bh(&dev->rx_lock); 314 goto out; 315 } 316 317 mt76_tx_status_lock(dev, &list); 318 cb->jiffies = jiffies; 319 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list); 320 mt76_tx_status_unlock(dev, &list); 321 322 out: 323 rcu_read_unlock(); 324 } 325 EXPORT_SYMBOL_GPL(__mt76_tx_complete_skb); 326 327 static int 328 __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, 329 struct mt76_wcid *wcid, struct ieee80211_sta *sta, 330 bool *stop) 331 { 332 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 333 struct mt76_queue *q = phy->q_tx[qid]; 334 struct mt76_dev *dev = phy->dev; 335 bool non_aql; 336 int pending; 337 int idx; 338 339 non_aql = !info->tx_time_est; 340 idx = dev->queue_ops->tx_queue_skb(phy, q, qid, skb, wcid, sta); 341 if (idx < 0 || !sta) 342 return idx; 343 344 q->entry[idx].wcid = wcid->idx; 345 346 if (!non_aql) 347 return idx; 348 349 /* the hardware can report the completion on a different link */ 350 pending = atomic_inc_return(&mt76_wcid_primary(wcid)->non_aql_packets); 351 if (stop && pending >= MT_MAX_NON_AQL_PKT) 352 *stop = true; 353 354 return idx; 355 } 356 357 void 358 mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta, 359 struct mt76_wcid *wcid, struct sk_buff *skb) 360 { 361 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 362 struct ieee80211_hdr *hdr = (void *)skb->data; 363 struct sk_buff_head *head; 364 365 if (mt76_testmode_enabled(phy)) { 366 ieee80211_free_txskb(phy->hw, skb); 367 return; 368 } 369 370 if (WARN_ON(skb_get_queue_mapping(skb) >= MT_TXQ_PSD)) 371 skb_set_queue_mapping(skb, MT_TXQ_BE); 372 373 if (wcid && !(wcid->tx_info & MT_WCID_TX_INFO_SET)) 374 ieee80211_get_tx_rates(info->control.vif, sta, skb, 375 info->control.rates, 1); 376 377 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 378 379 if ((info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) || 380 ((info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK) && 381 ieee80211_is_probe_req(hdr->frame_control))) 382 head = &wcid->tx_offchannel; 383 else 384 head = &wcid->tx_pending; 385 386 spin_lock_bh(&head->lock); 387 __skb_queue_tail(head, skb); 388 spin_unlock_bh(&head->lock); 389 390 spin_lock_bh(&phy->tx_lock); 391 if (list_empty(&wcid->tx_list)) 392 list_add_tail(&wcid->tx_list, &phy->tx_list); 393 spin_unlock_bh(&phy->tx_lock); 394 395 mt76_worker_schedule(&phy->dev->tx_worker); 396 } 397 EXPORT_SYMBOL_GPL(mt76_tx); 398 399 static struct sk_buff * 400 mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq) 401 { 402 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 403 struct ieee80211_tx_info *info; 404 struct sk_buff *skb; 405 406 skb = ieee80211_tx_dequeue(phy->hw, txq); 407 if (!skb) 408 return NULL; 409 410 info = IEEE80211_SKB_CB(skb); 411 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 412 413 return skb; 414 } 415 416 static void 417 mt76_queue_ps_skb(struct mt76_phy *phy, struct ieee80211_sta *sta, 418 struct sk_buff *skb, bool last, 419 enum ieee80211_frame_release_type reason) 420 { 421 struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; 422 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 423 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 424 425 info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE; 426 if (last) { 427 info->flags |= IEEE80211_TX_STATUS_EOSP | 428 IEEE80211_TX_CTL_REQ_TX_STATUS; 429 430 if (reason == IEEE80211_FRAME_RELEASE_UAPSD && 431 ieee80211_is_data_qos(hdr->frame_control)) 432 *ieee80211_get_qos_ctl(hdr) |= IEEE80211_QOS_CTL_EOSP; 433 } 434 435 mt76_skb_set_moredata(skb, !last); 436 __mt76_tx_queue_skb(phy, MT_TXQ_PSD, skb, wcid, sta, NULL); 437 } 438 439 void 440 mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, 441 u16 tids, int nframes, 442 enum ieee80211_frame_release_type reason, 443 bool more_data) 444 { 445 struct mt76_phy *phy = hw->priv; 446 struct mt76_dev *dev = phy->dev; 447 struct sk_buff *last_skb = NULL; 448 struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD]; 449 int i; 450 451 spin_lock_bh(&hwq->lock); 452 for (i = 0; tids && nframes; i++, tids >>= 1) { 453 struct ieee80211_txq *txq = sta->txq[i]; 454 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 455 struct sk_buff *skb; 456 457 if (!(tids & 1)) 458 continue; 459 460 do { 461 skb = mt76_txq_dequeue(phy, mtxq); 462 if (!skb) 463 break; 464 465 nframes--; 466 if (last_skb) 467 mt76_queue_ps_skb(phy, sta, last_skb, false, 468 reason); 469 470 last_skb = skb; 471 } while (nframes); 472 } 473 474 if (last_skb) { 475 mt76_queue_ps_skb(phy, sta, last_skb, true, reason); 476 dev->queue_ops->kick(dev, hwq); 477 } else { 478 ieee80211_sta_eosp(sta); 479 } 480 481 spin_unlock_bh(&hwq->lock); 482 } 483 EXPORT_SYMBOL_GPL(mt76_release_buffered_frames); 484 485 static bool 486 mt76_txq_stopped(struct mt76_queue *q) 487 { 488 return q->stopped || q->blocked || 489 q->queued + MT_TXQ_FREE_THR >= q->ndesc; 490 } 491 492 static int 493 mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, 494 struct mt76_txq *mtxq, struct mt76_wcid *wcid) 495 { 496 struct mt76_dev *dev = phy->dev; 497 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 498 enum mt76_txq_id qid = mt76_txq_get_qid(txq); 499 struct ieee80211_tx_info *info; 500 struct sk_buff *skb; 501 int n_frames = 1; 502 bool stop = false; 503 int idx; 504 505 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 506 if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) 507 return 0; 508 if (ieee80211_txq_aql_pending(phy->hw, txq)) 509 return 0; 510 } 511 512 if (atomic_read(&mt76_wcid_primary(wcid)->non_aql_packets) >= 513 MT_MAX_NON_AQL_PKT) 514 return 0; 515 516 skb = mt76_txq_dequeue(phy, mtxq); 517 if (!skb) 518 return 0; 519 520 info = IEEE80211_SKB_CB(skb); 521 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 522 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 523 info->control.rates, 1); 524 525 spin_lock(&q->lock); 526 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 527 spin_unlock(&q->lock); 528 if (idx < 0) 529 return idx; 530 531 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 532 goto out; 533 534 do { 535 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 536 break; 537 538 if (stop || mt76_txq_stopped(q)) 539 break; 540 541 skb = mt76_txq_dequeue(phy, mtxq); 542 if (!skb) 543 break; 544 545 info = IEEE80211_SKB_CB(skb); 546 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 547 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 548 info->control.rates, 1); 549 550 spin_lock(&q->lock); 551 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 552 spin_unlock(&q->lock); 553 if (idx < 0) 554 break; 555 556 n_frames++; 557 } while (1); 558 559 out: 560 spin_lock(&q->lock); 561 dev->queue_ops->kick(dev, q); 562 spin_unlock(&q->lock); 563 564 return n_frames; 565 } 566 567 static int 568 mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) 569 { 570 struct mt76_dev *dev = phy->dev; 571 struct ieee80211_txq *txq; 572 struct mt76_txq *mtxq; 573 struct mt76_wcid *wcid; 574 struct mt76_queue *q; 575 int ret = 0; 576 577 while (1) { 578 int n_frames = 0; 579 580 txq = ieee80211_next_txq(phy->hw, qid); 581 if (!txq) 582 break; 583 584 mtxq = (struct mt76_txq *)txq->drv_priv; 585 wcid = __mt76_wcid_ptr(dev, mtxq->wcid); 586 if (!wcid) 587 continue; 588 589 phy = mt76_dev_phy(dev, wcid->phy_idx); 590 if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) 591 continue; 592 593 q = phy->q_tx[qid]; 594 595 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { 596 if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) 597 continue; 598 599 if (!mt76_txq_stopped(q)) 600 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 601 602 ieee80211_return_txq(phy->hw, txq, false); 603 604 if (unlikely(n_frames < 0)) 605 return n_frames; 606 ret += n_frames; 607 continue; 608 } 609 610 if (atomic_read(&mt76_wcid_primary(wcid)->non_aql_packets) >= 611 MT_MAX_NON_AQL_PKT) 612 continue; 613 if (dev->queue_ops->tx_cleanup && 614 q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { 615 dev->queue_ops->tx_cleanup(dev, q, false); 616 } 617 618 if (mtxq->send_bar && mtxq->aggr) { 619 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 620 struct ieee80211_sta *sta = txq->sta; 621 struct ieee80211_vif *vif = txq->vif; 622 u16 agg_ssn = mtxq->agg_ssn; 623 u8 tid = txq->tid; 624 625 mtxq->send_bar = false; 626 ieee80211_send_bar(vif, sta->addr, tid, agg_ssn); 627 } 628 629 if (!mt76_txq_stopped(q)) 630 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 631 632 ieee80211_return_txq(phy->hw, txq, false); 633 634 if (unlikely(n_frames < 0)) 635 return n_frames; 636 637 ret += n_frames; 638 } 639 640 return ret; 641 } 642 643 void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid) 644 { 645 int len; 646 647 if (qid >= 4) 648 return; 649 650 local_bh_disable(); 651 rcu_read_lock(); 652 653 do { 654 ieee80211_txq_schedule_start(phy->hw, qid); 655 len = mt76_txq_schedule_list(phy, qid); 656 ieee80211_txq_schedule_end(phy->hw, qid); 657 } while (len > 0); 658 659 rcu_read_unlock(); 660 local_bh_enable(); 661 } 662 EXPORT_SYMBOL_GPL(mt76_txq_schedule); 663 664 static int 665 mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, 666 struct sk_buff_head *head) 667 { 668 struct mt76_dev *dev = phy->dev; 669 struct ieee80211_sta *sta; 670 struct mt76_queue *q; 671 struct sk_buff *skb; 672 int ret = 0; 673 674 spin_lock(&head->lock); 675 while ((skb = skb_peek(head)) != NULL) { 676 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 677 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 678 int qid = skb_get_queue_mapping(skb); 679 680 if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) && 681 !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && 682 !ieee80211_is_data_present(hdr->frame_control) && 683 (!(wcid->tx_info & MT_WCID_TX_INFO_SET) || 684 !ieee80211_is_bufferable_mmpdu(skb) || 685 ieee80211_is_deauth(hdr->frame_control) || 686 ieee80211_is_disassoc(hdr->frame_control) || 687 head == &wcid->tx_offchannel)) 688 qid = MT_TXQ_PSD; 689 690 q = phy->q_tx[qid]; 691 if (mt76_txq_stopped(q) || test_bit(MT76_RESET, &phy->state)) { 692 ret = -1; 693 break; 694 } 695 696 __skb_unlink(skb, head); 697 spin_unlock(&head->lock); 698 699 sta = wcid_to_sta(wcid); 700 spin_lock(&q->lock); 701 __mt76_tx_queue_skb(phy, qid, skb, wcid, sta, NULL); 702 dev->queue_ops->kick(dev, q); 703 spin_unlock(&q->lock); 704 705 spin_lock(&head->lock); 706 } 707 spin_unlock(&head->lock); 708 709 return ret; 710 } 711 712 void mt76_txq_schedule_pending(struct mt76_phy *phy) 713 { 714 LIST_HEAD(tx_list); 715 int ret = 0; 716 717 if (list_empty(&phy->tx_list)) 718 return; 719 720 local_bh_disable(); 721 rcu_read_lock(); 722 723 spin_lock(&phy->tx_lock); 724 list_splice_init(&phy->tx_list, &tx_list); 725 while (!list_empty(&tx_list)) { 726 struct mt76_wcid *wcid; 727 728 wcid = list_first_entry(&tx_list, struct mt76_wcid, tx_list); 729 list_del_init(&wcid->tx_list); 730 731 spin_unlock(&phy->tx_lock); 732 if (ret >= 0) 733 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel); 734 if (ret >= 0 && !phy->offchannel) 735 ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); 736 spin_lock(&phy->tx_lock); 737 738 if ((!skb_queue_empty(&wcid->tx_pending) || 739 !skb_queue_empty(&wcid->tx_offchannel)) && 740 list_empty(&wcid->tx_list)) 741 list_add_tail(&wcid->tx_list, &phy->tx_list); 742 } 743 spin_unlock(&phy->tx_lock); 744 745 rcu_read_unlock(); 746 local_bh_enable(); 747 } 748 749 void mt76_txq_schedule_all(struct mt76_phy *phy) 750 { 751 struct mt76_phy *main_phy = &phy->dev->phy; 752 int i; 753 754 mt76_txq_schedule_pending(phy); 755 756 if (phy != main_phy && phy->hw == main_phy->hw) 757 return; 758 759 for (i = 0; i <= MT_TXQ_BK; i++) 760 mt76_txq_schedule(phy, i); 761 } 762 EXPORT_SYMBOL_GPL(mt76_txq_schedule_all); 763 764 void mt76_tx_worker_run(struct mt76_dev *dev) 765 { 766 struct mt76_phy *phy; 767 int i; 768 769 mt76_txq_schedule_all(&dev->phy); 770 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 771 phy = dev->phys[i]; 772 if (!phy) 773 continue; 774 775 mt76_txq_schedule_all(phy); 776 } 777 778 #ifdef CONFIG_NL80211_TESTMODE 779 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 780 phy = dev->phys[i]; 781 if (!phy || !phy->test.tx_pending) 782 continue; 783 784 mt76_testmode_tx_pending(phy); 785 } 786 #endif 787 } 788 EXPORT_SYMBOL_GPL(mt76_tx_worker_run); 789 790 void mt76_tx_worker(struct mt76_worker *w) 791 { 792 struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker); 793 794 mt76_tx_worker_run(dev); 795 } 796 797 void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta, 798 bool send_bar) 799 { 800 int i; 801 802 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 803 struct ieee80211_txq *txq = sta->txq[i]; 804 struct mt76_queue *hwq; 805 struct mt76_txq *mtxq; 806 807 if (!txq) 808 continue; 809 810 hwq = phy->q_tx[mt76_txq_get_qid(txq)]; 811 mtxq = (struct mt76_txq *)txq->drv_priv; 812 813 spin_lock_bh(&hwq->lock); 814 mtxq->send_bar = mtxq->aggr && send_bar; 815 spin_unlock_bh(&hwq->lock); 816 } 817 } 818 EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); 819 820 void mt76_sta_ps_transition(struct mt76_dev *dev, struct mt76_wcid *wcid, 821 bool ps) 822 { 823 struct ieee80211_sta *sta; 824 struct ieee80211_hw *hw; 825 int i; 826 827 if (ps) { 828 set_bit(MT_WCID_FLAG_PS, &wcid->flags); 829 mt76_worker_schedule(&dev->tx_worker); 830 return; 831 } 832 833 clear_bit(MT_WCID_FLAG_PS, &wcid->flags); 834 835 sta = wcid_to_sta(wcid); 836 if (!sta) 837 return; 838 839 hw = mt76_phy_hw(dev, wcid->phy_idx); 840 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) 841 if (sta->txq[i]) 842 ieee80211_schedule_txq(hw, sta->txq[i]); 843 mt76_worker_schedule(&dev->tx_worker); 844 } 845 EXPORT_SYMBOL_GPL(mt76_sta_ps_transition); 846 847 void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) 848 { 849 struct mt76_phy *phy = hw->priv; 850 struct mt76_dev *dev = phy->dev; 851 852 mt76_worker_schedule(&dev->tx_worker); 853 } 854 EXPORT_SYMBOL_GPL(mt76_wake_tx_queue); 855 856 u8 mt76_ac_to_hwq(u8 ac) 857 { 858 static const u8 wmm_queue_map[] = { 859 [IEEE80211_AC_BE] = 0, 860 [IEEE80211_AC_BK] = 1, 861 [IEEE80211_AC_VI] = 2, 862 [IEEE80211_AC_VO] = 3, 863 }; 864 865 if (WARN_ON(ac >= IEEE80211_NUM_ACS)) 866 return 0; 867 868 return wmm_queue_map[ac]; 869 } 870 EXPORT_SYMBOL_GPL(mt76_ac_to_hwq); 871 872 int mt76_skb_adjust_pad(struct sk_buff *skb, int pad) 873 { 874 struct sk_buff *iter, *last = skb; 875 876 /* First packet of a A-MSDU burst keeps track of the whole burst 877 * length, need to update length of it and the last packet. 878 */ 879 skb_walk_frags(skb, iter) { 880 last = iter; 881 if (!iter->next) { 882 skb->data_len += pad; 883 skb->len += pad; 884 break; 885 } 886 } 887 888 if (skb_pad(last, pad)) 889 return -ENOMEM; 890 891 __skb_put(last, pad); 892 893 return 0; 894 } 895 EXPORT_SYMBOL_GPL(mt76_skb_adjust_pad); 896 897 void mt76_queue_tx_complete(struct mt76_dev *dev, struct mt76_queue *q, 898 struct mt76_queue_entry *e) 899 { 900 if (e->skb) 901 dev->drv->tx_complete_skb(dev, e); 902 903 spin_lock_bh(&q->lock); 904 q->tail = (q->tail + 1) % q->ndesc; 905 q->queued--; 906 spin_unlock_bh(&q->lock); 907 } 908 EXPORT_SYMBOL_GPL(mt76_queue_tx_complete); 909 910 void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked) 911 { 912 struct mt76_phy *phy = &dev->phy; 913 struct mt76_queue *q = phy->q_tx[0]; 914 915 if (blocked == q->blocked) 916 return; 917 918 q->blocked = blocked; 919 920 phy = dev->phys[MT_BAND1]; 921 if (phy) { 922 q = phy->q_tx[0]; 923 q->blocked = blocked; 924 } 925 phy = dev->phys[MT_BAND2]; 926 if (phy) { 927 q = phy->q_tx[0]; 928 q->blocked = blocked; 929 } 930 931 if (!blocked) 932 mt76_worker_schedule(&dev->tx_worker); 933 } 934 EXPORT_SYMBOL_GPL(__mt76_set_tx_blocked); 935 936 int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) 937 { 938 int token; 939 940 spin_lock_bh(&dev->token_lock); 941 942 token = idr_alloc(&dev->token, *ptxwi, dev->token_start, 943 dev->token_start + dev->token_size, 944 GFP_ATOMIC); 945 if (token >= dev->token_start) { 946 dev->token_count++; 947 948 if ((*ptxwi)->qid == MT_TXQ_PSD) { 949 struct mt76_phy *mphy = mt76_dev_phy(dev, (*ptxwi)->phy_idx); 950 atomic_inc(&mphy->mgmt_tx_pending); 951 } 952 } 953 954 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 955 if (mtk_wed_device_active(&dev->mmio.wed) && 956 token >= dev->mmio.wed.wlan.token_start) 957 dev->wed_token_count++; 958 #endif 959 960 if (dev->token_count >= dev->token_size - MT76_TOKEN_FREE_THR) 961 __mt76_set_tx_blocked(dev, true); 962 963 spin_unlock_bh(&dev->token_lock); 964 965 return token; 966 } 967 EXPORT_SYMBOL_GPL(mt76_token_consume); 968 969 int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr, 970 struct mt76_txwi_cache *t, dma_addr_t phys) 971 { 972 int token; 973 974 spin_lock_bh(&dev->rx_token_lock); 975 token = idr_alloc(&dev->rx_token, t, 0, dev->rx_token_size, 976 GFP_ATOMIC); 977 if (token >= 0) { 978 t->ptr = ptr; 979 t->dma_addr = phys; 980 } 981 spin_unlock_bh(&dev->rx_token_lock); 982 983 return token; 984 } 985 EXPORT_SYMBOL_GPL(mt76_rx_token_consume); 986 987 struct mt76_txwi_cache * 988 mt76_token_release(struct mt76_dev *dev, int token, bool *wake) 989 { 990 struct mt76_txwi_cache *txwi; 991 992 spin_lock_bh(&dev->token_lock); 993 994 txwi = idr_remove(&dev->token, token); 995 if (txwi) { 996 dev->token_count--; 997 998 if (txwi->qid == MT_TXQ_PSD) { 999 struct mt76_phy *mphy = mt76_dev_phy(dev, txwi->phy_idx); 1000 if (atomic_dec_and_test(&mphy->mgmt_tx_pending)) 1001 wake_up(&dev->tx_wait); 1002 } 1003 1004 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 1005 if (mtk_wed_device_active(&dev->mmio.wed) && 1006 token >= dev->mmio.wed.wlan.token_start && 1007 --dev->wed_token_count == 0) 1008 wake_up(&dev->tx_wait); 1009 #endif 1010 } 1011 1012 if (wake && dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && 1013 dev->phy.q_tx[0]->blocked) 1014 *wake = true; 1015 1016 spin_unlock_bh(&dev->token_lock); 1017 1018 return txwi; 1019 } 1020 EXPORT_SYMBOL_GPL(mt76_token_release); 1021 1022 struct mt76_txwi_cache * 1023 mt76_rx_token_release(struct mt76_dev *dev, int token) 1024 { 1025 struct mt76_txwi_cache *t; 1026 1027 spin_lock_bh(&dev->rx_token_lock); 1028 t = idr_remove(&dev->rx_token, token); 1029 spin_unlock_bh(&dev->rx_token_lock); 1030 1031 return t; 1032 } 1033 EXPORT_SYMBOL_GPL(mt76_rx_token_release); 1034