1 // SPDX-License-Identifier: ISC
11 	if (!txq->sta)  in mt76_txq_get_qid()
14 	return txq->ac;  in mt76_txq_get_qid()
20 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;  in mt76_tx_check_agg_ssn()
25 	if (!sta || !ieee80211_is_data_qos(hdr->frame_control) ||  in mt76_tx_check_agg_ssn()
26 	    !ieee80211_is_data_present(hdr->frame_control))  in mt76_tx_check_agg_ssn()
29 	tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK;  in mt76_tx_check_agg_ssn()
30 	txq = sta->txq[tid];  in mt76_tx_check_agg_ssn()
31 	mtxq = (struct mt76_txq *)txq->drv_priv;  in mt76_tx_check_agg_ssn()
32 	if (!mtxq->aggr)  in mt76_tx_check_agg_ssn()
35 	mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10;  in mt76_tx_check_agg_ssn()
41 		   __acquires(&dev->status_lock)  in mt76_tx_status_lock()
44 	spin_lock_bh(&dev->status_lock);  in mt76_tx_status_lock()
50 		      __releases(&dev->status_lock)  in mt76_tx_status_unlock()
55 	spin_unlock_bh(&dev->status_lock);  in mt76_tx_status_unlock()
59 		struct ieee80211_tx_status status = {  in mt76_tx_status_unlock()  local
67 		wcid = __mt76_wcid_ptr(dev, cb->wcid);  in mt76_tx_status_unlock()
69 			status.sta = wcid_to_sta(wcid);  in mt76_tx_status_unlock()
70 			if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) {  in mt76_tx_status_unlock()
71 				rs.rate_idx = wcid->rate;  in mt76_tx_status_unlock()
72 				status.rates = &rs;  in mt76_tx_status_unlock()
73 				status.n_rates = 1;  in mt76_tx_status_unlock()
75 				status.n_rates = 0;  in mt76_tx_status_unlock()
80 		spin_lock_bh(&dev->rx_lock);  in mt76_tx_status_unlock()
81 		ieee80211_tx_status_ext(hw, &status);  in mt76_tx_status_unlock()
82 		spin_unlock_bh(&dev->rx_lock);  in mt76_tx_status_unlock()
96 	flags |= cb->flags;  in __mt76_tx_status_skb_done()
97 	cb->flags = flags;  in __mt76_tx_status_skb_done()
102 	/* Tx status can be unreliable. if it fails, mark the frame as ACKed */  in __mt76_tx_status_skb_done()
104 	    (dev->drv->drv_flags & MT_DRV_IGNORE_TXS_FAILED)) {  in __mt76_tx_status_skb_done()
105 		info->status.rates[0].count = 0;  in __mt76_tx_status_skb_done()
106 		info->status.rates[0].idx = -1;  in __mt76_tx_status_skb_done()
107 		info->flags |= IEEE80211_TX_STAT_ACK;  in __mt76_tx_status_skb_done()
125 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;  in mt76_tx_status_skb_add()
132 	if (!wcid || !rcu_access_pointer(dev->wcid[wcid->idx]))  in mt76_tx_status_skb_add()
135 	if (info->flags & IEEE80211_TX_CTL_NO_ACK)  in mt76_tx_status_skb_add()
138 	if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS |  in mt76_tx_status_skb_add()
140 		if (mtk_wed_device_active(&dev->mmio.wed) &&  in mt76_tx_status_skb_add()
141 		    ((info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) ||  in mt76_tx_status_skb_add()
142 		     ieee80211_is_data(hdr->frame_control)))  in mt76_tx_status_skb_add()
148 	spin_lock_bh(&dev->status_lock);  in mt76_tx_status_skb_add()
150 	pid = idr_alloc(&wcid->pktid, skb, MT_PACKET_ID_FIRST,  in mt76_tx_status_skb_add()
157 	cb->wcid = wcid->idx;  in mt76_tx_status_skb_add()
158 	cb->pktid = pid;  in mt76_tx_status_skb_add()
160 	if (list_empty(&wcid->list))  in mt76_tx_status_skb_add()
161 		list_add_tail(&wcid->list, &dev->wcid_list);  in mt76_tx_status_skb_add()
164 	spin_unlock_bh(&dev->status_lock);  in mt76_tx_status_skb_add()
177 	lockdep_assert_held(&dev->status_lock);  in mt76_tx_status_skb_get()
179 	skb = idr_remove(&wcid->pktid, pktid);  in mt76_tx_status_skb_get()
184 	idr_for_each_entry(&wcid->pktid, skb, id) {  in mt76_tx_status_skb_get()
188 			if (!(cb->flags & MT_TX_CB_DMA_DONE))  in mt76_tx_status_skb_get()
191 			if (time_is_after_jiffies(cb->jiffies +  in mt76_tx_status_skb_get()
199 		idr_remove(&wcid->pktid, cb->pktid);  in mt76_tx_status_skb_get()
205 	if (idr_is_empty(&wcid->pktid))  in mt76_tx_status_skb_get()
206 		list_del_init(&wcid->list);  in mt76_tx_status_skb_get()
219 	list_for_each_entry_safe(wcid, tmp, &dev->wcid_list, list)  in mt76_tx_status_check()
220 		mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list);  in mt76_tx_status_check()
232 	if (!wcid || info->tx_time_est)  in mt76_tx_check_non_aql()
235 	pending = atomic_dec_return(&wcid->non_aql_packets);  in mt76_tx_check_non_aql()
237 		atomic_cmpxchg(&wcid->non_aql_packets, pending, 0);  in mt76_tx_check_non_aql()
244 	struct ieee80211_tx_status status = {  in __mt76_tx_complete_skb()  local
259 		struct mt76_phy *phy = hw->priv;  in __mt76_tx_complete_skb()
261 		if (skb == phy->test.tx_skb)  in __mt76_tx_complete_skb()
262 			phy->test.tx_done++;  in __mt76_tx_complete_skb()
263 		if (phy->test.tx_queued == phy->test.tx_done)  in __mt76_tx_complete_skb()
264 			wake_up(&dev->tx_wait);  in __mt76_tx_complete_skb()
271 	if (cb->pktid < MT_PACKET_ID_FIRST) {  in __mt76_tx_complete_skb()
275 		status.sta = wcid_to_sta(wcid);  in __mt76_tx_complete_skb()
276 		if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) {  in __mt76_tx_complete_skb()
277 			rs.rate_idx = wcid->rate;  in __mt76_tx_complete_skb()
278 			status.rates = &rs;  in __mt76_tx_complete_skb()
279 			status.n_rates = 1;  in __mt76_tx_complete_skb()
281 		spin_lock_bh(&dev->rx_lock);  in __mt76_tx_complete_skb()
282 		ieee80211_tx_status_ext(hw, &status);  in __mt76_tx_complete_skb()
283 		spin_unlock_bh(&dev->rx_lock);  in __mt76_tx_complete_skb()
288 	cb->jiffies = jiffies;  in __mt76_tx_complete_skb()
303 	struct mt76_queue *q = phy->q_tx[qid];  in __mt76_tx_queue_skb()
304 	struct mt76_dev *dev = phy->dev;  in __mt76_tx_queue_skb()
309 	non_aql = !info->tx_time_est;  in __mt76_tx_queue_skb()
310 	idx = dev->queue_ops->tx_queue_skb(phy, q, qid, skb, wcid, sta);  in __mt76_tx_queue_skb()
314 	wcid = (struct mt76_wcid *)sta->drv_priv;  in __mt76_tx_queue_skb()
315 	if (!wcid->sta)  in __mt76_tx_queue_skb()
318 	q->entry[idx].wcid = wcid->idx;  in __mt76_tx_queue_skb()
323 	pending = atomic_inc_return(&wcid->non_aql_packets);  in __mt76_tx_queue_skb()
335 	struct ieee80211_hdr *hdr = (void *)skb->data;  in mt76_tx()
339 		ieee80211_free_txskb(phy->hw, skb);  in mt76_tx()
346 	if (wcid && !(wcid->tx_info & MT_WCID_TX_INFO_SET))  in mt76_tx()
347 		ieee80211_get_tx_rates(info->control.vif, sta, skb,  in mt76_tx()
348 				       info->control.rates, 1);  in mt76_tx()
350 	info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx);  in mt76_tx()
352 	if ((info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) ||  in mt76_tx()
353 	    ((info->control.flags & IEEE80211_TX_CTRL_DONT_USE_RATE_MASK) &&  in mt76_tx()
354 	     ieee80211_is_probe_req(hdr->frame_control)))  in mt76_tx()
355 		head = &wcid->tx_offchannel;  in mt76_tx()
357 		head = &wcid->tx_pending;  in mt76_tx()
359 	spin_lock_bh(&head->lock);  in mt76_tx()
361 	spin_unlock_bh(&head->lock);  in mt76_tx()
363 	spin_lock_bh(&phy->tx_lock);  in mt76_tx()
364 	if (list_empty(&wcid->tx_list))  in mt76_tx()
365 		list_add_tail(&wcid->tx_list, &phy->tx_list);  in mt76_tx()
366 	spin_unlock_bh(&phy->tx_lock);  in mt76_tx()
368 	mt76_worker_schedule(&phy->dev->tx_worker);  in mt76_tx()
379 	skb = ieee80211_tx_dequeue(phy->hw, txq);  in mt76_txq_dequeue()
384 	info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx);  in mt76_txq_dequeue()
393 	struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv;  in mt76_queue_ps_skb()
396 	info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;  in mt76_queue_ps_skb()
398 		info->flags |= IEEE80211_TX_STATUS_EOSP |  in mt76_queue_ps_skb()
411 	struct mt76_phy *phy = hw->priv;  in mt76_release_buffered_frames()
412 	struct mt76_dev *dev = phy->dev;  in mt76_release_buffered_frames()
414 	struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD];  in mt76_release_buffered_frames()
417 	spin_lock_bh(&hwq->lock);  in mt76_release_buffered_frames()
419 		struct ieee80211_txq *txq = sta->txq[i];  in mt76_release_buffered_frames()
420 		struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv;  in mt76_release_buffered_frames()
431 			nframes--;  in mt76_release_buffered_frames()
441 		dev->queue_ops->kick(dev, hwq);  in mt76_release_buffered_frames()
446 	spin_unlock_bh(&hwq->lock);  in mt76_release_buffered_frames()
453 	return q->stopped || q->blocked ||  in mt76_txq_stopped()
454 	       q->queued + MT_TXQ_FREE_THR >= q->ndesc;  in mt76_txq_stopped()
461 	struct mt76_dev *dev = phy->dev;  in mt76_txq_send_burst()
470 	if (test_bit(MT_WCID_FLAG_PS, &wcid->flags))  in mt76_txq_send_burst()
473 	if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT)  in mt76_txq_send_burst()
481 	if (!(wcid->tx_info & MT_WCID_TX_INFO_SET))  in mt76_txq_send_burst()
482 		ieee80211_get_tx_rates(txq->vif, txq->sta, skb,  in mt76_txq_send_burst()
483 				       info->control.rates, 1);  in mt76_txq_send_burst()
485 	spin_lock(&q->lock);  in mt76_txq_send_burst()
486 	idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop);  in mt76_txq_send_burst()
487 	spin_unlock(&q->lock);  in mt76_txq_send_burst()
492 		if (test_bit(MT76_RESET, &phy->state) || phy->offchannel)  in mt76_txq_send_burst()
503 		if (!(wcid->tx_info & MT_WCID_TX_INFO_SET))  in mt76_txq_send_burst()
504 			ieee80211_get_tx_rates(txq->vif, txq->sta, skb,  in mt76_txq_send_burst()
505 					       info->control.rates, 1);  in mt76_txq_send_burst()
507 		spin_lock(&q->lock);  in mt76_txq_send_burst()
508 		idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop);  in mt76_txq_send_burst()
509 		spin_unlock(&q->lock);  in mt76_txq_send_burst()
516 	spin_lock(&q->lock);  in mt76_txq_send_burst()
517 	dev->queue_ops->kick(dev, q);  in mt76_txq_send_burst()
518 	spin_unlock(&q->lock);  in mt76_txq_send_burst()
526 	struct mt76_dev *dev = phy->dev;  in mt76_txq_schedule_list()
536 		txq = ieee80211_next_txq(phy->hw, qid);  in mt76_txq_schedule_list()
540 		mtxq = (struct mt76_txq *)txq->drv_priv;  in mt76_txq_schedule_list()
541 		wcid = __mt76_wcid_ptr(dev, mtxq->wcid);  in mt76_txq_schedule_list()
542 		if (!wcid || test_bit(MT_WCID_FLAG_PS, &wcid->flags))  in mt76_txq_schedule_list()
545 		phy = mt76_dev_phy(dev, wcid->phy_idx);  in mt76_txq_schedule_list()
546 		if (test_bit(MT76_RESET, &phy->state) || phy->offchannel)  in mt76_txq_schedule_list()
549 		q = phy->q_tx[qid];  in mt76_txq_schedule_list()
550 		if (dev->queue_ops->tx_cleanup &&  in mt76_txq_schedule_list()
551 		    q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) {  in mt76_txq_schedule_list()
552 			dev->queue_ops->tx_cleanup(dev, q, false);  in mt76_txq_schedule_list()
555 		if (mtxq->send_bar && mtxq->aggr) {  in mt76_txq_schedule_list()
557 			struct ieee80211_sta *sta = txq->sta;  in mt76_txq_schedule_list()
558 			struct ieee80211_vif *vif = txq->vif;  in mt76_txq_schedule_list()
559 			u16 agg_ssn = mtxq->agg_ssn;  in mt76_txq_schedule_list()
560 			u8 tid = txq->tid;  in mt76_txq_schedule_list()
562 			mtxq->send_bar = false;  in mt76_txq_schedule_list()
563 			ieee80211_send_bar(vif, sta->addr, tid, agg_ssn);  in mt76_txq_schedule_list()
569 		ieee80211_return_txq(phy->hw, txq, false);  in mt76_txq_schedule_list()
591 		ieee80211_txq_schedule_start(phy->hw, qid);  in mt76_txq_schedule()
593 		ieee80211_txq_schedule_end(phy->hw, qid);  in mt76_txq_schedule()
605 	struct mt76_dev *dev = phy->dev;  in mt76_txq_schedule_pending_wcid()
611 	spin_lock(&head->lock);  in mt76_txq_schedule_pending_wcid()
613 		struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;  in mt76_txq_schedule_pending_wcid()
617 		if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) &&  in mt76_txq_schedule_pending_wcid()
618 		    !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&  in mt76_txq_schedule_pending_wcid()
619 		    !ieee80211_is_data(hdr->frame_control) &&  in mt76_txq_schedule_pending_wcid()
621 		     ieee80211_is_deauth(hdr->frame_control)))  in mt76_txq_schedule_pending_wcid()
624 		q = phy->q_tx[qid];  in mt76_txq_schedule_pending_wcid()
625 		if (mt76_txq_stopped(q) || test_bit(MT76_RESET, &phy->state)) {  in mt76_txq_schedule_pending_wcid()
626 			ret = -1;  in mt76_txq_schedule_pending_wcid()
631 		spin_unlock(&head->lock);  in mt76_txq_schedule_pending_wcid()
634 		spin_lock(&q->lock);  in mt76_txq_schedule_pending_wcid()
636 		dev->queue_ops->kick(dev, q);  in mt76_txq_schedule_pending_wcid()
637 		spin_unlock(&q->lock);  in mt76_txq_schedule_pending_wcid()
639 		spin_lock(&head->lock);  in mt76_txq_schedule_pending_wcid()
641 	spin_unlock(&head->lock);  in mt76_txq_schedule_pending_wcid()
651 	if (list_empty(&phy->tx_list))  in mt76_txq_schedule_pending()
657 	spin_lock(&phy->tx_lock);  in mt76_txq_schedule_pending()
658 	list_splice_init(&phy->tx_list, &tx_list);  in mt76_txq_schedule_pending()
663 		list_del_init(&wcid->tx_list);  in mt76_txq_schedule_pending()
665 		spin_unlock(&phy->tx_lock);  in mt76_txq_schedule_pending()
667 			ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_offchannel);  in mt76_txq_schedule_pending()
668 		if (ret >= 0 && !phy->offchannel)  in mt76_txq_schedule_pending()
669 			ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending);  in mt76_txq_schedule_pending()
670 		spin_lock(&phy->tx_lock);  in mt76_txq_schedule_pending()
672 		if (!skb_queue_empty(&wcid->tx_pending) &&  in mt76_txq_schedule_pending()
673 		    !skb_queue_empty(&wcid->tx_offchannel) &&  in mt76_txq_schedule_pending()
674 		    list_empty(&wcid->tx_list))  in mt76_txq_schedule_pending()
675 			list_add_tail(&wcid->tx_list, &phy->tx_list);  in mt76_txq_schedule_pending()
677 	spin_unlock(&phy->tx_lock);  in mt76_txq_schedule_pending()
685 	struct mt76_phy *main_phy = &phy->dev->phy;  in mt76_txq_schedule_all()
690 	if (phy != main_phy && phy->hw == main_phy->hw)  in mt76_txq_schedule_all()
703 	mt76_txq_schedule_all(&dev->phy);  in mt76_tx_worker_run()
704 	for (i = 0; i < ARRAY_SIZE(dev->phys); i++) {  in mt76_tx_worker_run()
705 		phy = dev->phys[i];  in mt76_tx_worker_run()
713 	for (i = 0; i < ARRAY_SIZE(dev->phys); i++) {  in mt76_tx_worker_run()
714 		phy = dev->phys[i];  in mt76_tx_worker_run()
715 		if (!phy || !phy->test.tx_pending)  in mt76_tx_worker_run()
736 	for (i = 0; i < ARRAY_SIZE(sta->txq); i++) {  in mt76_stop_tx_queues()
737 		struct ieee80211_txq *txq = sta->txq[i];  in mt76_stop_tx_queues()
744 		hwq = phy->q_tx[mt76_txq_get_qid(txq)];  in mt76_stop_tx_queues()
745 		mtxq = (struct mt76_txq *)txq->drv_priv;  in mt76_stop_tx_queues()
747 		spin_lock_bh(&hwq->lock);  in mt76_stop_tx_queues()
748 		mtxq->send_bar = mtxq->aggr && send_bar;  in mt76_stop_tx_queues()
749 		spin_unlock_bh(&hwq->lock);  in mt76_stop_tx_queues()
756 	struct mt76_phy *phy = hw->priv;  in mt76_wake_tx_queue()
757 	struct mt76_dev *dev = phy->dev;  in mt76_wake_tx_queue()
759 	mt76_worker_schedule(&dev->tx_worker);  in mt76_wake_tx_queue()
783 	/* First packet of a A-MSDU burst keeps track of the whole burst  in mt76_skb_adjust_pad()
788 		if (!iter->next) {  in mt76_skb_adjust_pad()
789 			skb->data_len += pad;  in mt76_skb_adjust_pad()
790 			skb->len += pad;  in mt76_skb_adjust_pad()
796 		return -ENOMEM;  in mt76_skb_adjust_pad()
807 	if (e->skb)  in mt76_queue_tx_complete()
808 		dev->drv->tx_complete_skb(dev, e);  in mt76_queue_tx_complete()
810 	spin_lock_bh(&q->lock);  in mt76_queue_tx_complete()
811 	q->tail = (q->tail + 1) % q->ndesc;  in mt76_queue_tx_complete()
812 	q->queued--;  in mt76_queue_tx_complete()
813 	spin_unlock_bh(&q->lock);  in mt76_queue_tx_complete()
819 	struct mt76_phy *phy = &dev->phy;  in __mt76_set_tx_blocked()
820 	struct mt76_queue *q = phy->q_tx[0];  in __mt76_set_tx_blocked()
822 	if (blocked == q->blocked)  in __mt76_set_tx_blocked()
825 	q->blocked = blocked;  in __mt76_set_tx_blocked()
827 	phy = dev->phys[MT_BAND1];  in __mt76_set_tx_blocked()
829 		q = phy->q_tx[0];  in __mt76_set_tx_blocked()
830 		q->blocked = blocked;  in __mt76_set_tx_blocked()
832 	phy = dev->phys[MT_BAND2];  in __mt76_set_tx_blocked()
834 		q = phy->q_tx[0];  in __mt76_set_tx_blocked()
835 		q->blocked = blocked;  in __mt76_set_tx_blocked()
839 		mt76_worker_schedule(&dev->tx_worker);  in __mt76_set_tx_blocked()
847 	spin_lock_bh(&dev->token_lock);  in mt76_token_consume()
849 	token = idr_alloc(&dev->token, *ptxwi, 0, dev->token_size, GFP_ATOMIC);  in mt76_token_consume()
851 		dev->token_count++;  in mt76_token_consume()
854 	if (mtk_wed_device_active(&dev->mmio.wed) &&  in mt76_token_consume()
855 	    token >= dev->mmio.wed.wlan.token_start)  in mt76_token_consume()
856 		dev->wed_token_count++;  in mt76_token_consume()
859 	if (dev->token_count >= dev->token_size - MT76_TOKEN_FREE_THR)  in mt76_token_consume()
862 	spin_unlock_bh(&dev->token_lock);  in mt76_token_consume()
873 	spin_lock_bh(&dev->rx_token_lock);  in mt76_rx_token_consume()
874 	token = idr_alloc(&dev->rx_token, t, 0, dev->rx_token_size,  in mt76_rx_token_consume()
877 		t->ptr = ptr;  in mt76_rx_token_consume()
878 		t->dma_addr = phys;  in mt76_rx_token_consume()
880 	spin_unlock_bh(&dev->rx_token_lock);  in mt76_rx_token_consume()
891 	spin_lock_bh(&dev->token_lock);  in mt76_token_release()
893 	txwi = idr_remove(&dev->token, token);  in mt76_token_release()
895 		dev->token_count--;  in mt76_token_release()
898 		if (mtk_wed_device_active(&dev->mmio.wed) &&  in mt76_token_release()
899 		    token >= dev->mmio.wed.wlan.token_start &&  in mt76_token_release()
900 		    --dev->wed_token_count == 0)  in mt76_token_release()
901 			wake_up(&dev->tx_wait);  in mt76_token_release()
905 	if (dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR &&  in mt76_token_release()
906 	    dev->phy.q_tx[0]->blocked)  in mt76_token_release()
909 	spin_unlock_bh(&dev->token_lock);  in mt76_token_release()
920 	spin_lock_bh(&dev->rx_token_lock);  in mt76_rx_token_release()
921 	t = idr_remove(&dev->rx_token, token);  in mt76_rx_token_release()
922 	spin_unlock_bh(&dev->rx_token_lock);  in mt76_rx_token_release()