Lines Matching +full:queue +full:- +full:rx
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (C) 2003-2014, 2018-2024 Intel Corporation
4 * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
5 * Copyright (C) 2016-2017 Intel Deutschland GmbH
11 #include "iwl-prph.h"
12 #include "iwl-io.h"
14 #include "iwl-op-mode.h"
15 #include "iwl-context-info-gen3.h"
19 * RX path functions
24 * Rx theory of operation
28 * used not only for Rx frames, but for any command response or notification
29 * from the NIC. The driver and NIC manage the Rx buffers by means
32 * Rx Queue Indexes
33 * The host/firmware share two index registers for managing the Rx buffers.
36 * to -- the driver can read up to (but not including) this position and get
40 * The WRITE index maps to the last position the driver has read from -- the
43 * The queue is empty (no good data) if WRITE = READ - 1, and is full if
46 * During initialization, the host sets up the READ queue position to the first
47 * INDEX position, and WRITE to the last (READ - 1 wrapped)
50 * and fire the RX interrupt. The driver can then query the READ index and
52 * resets the Rx queue buffers with new memory.
55 * + A list of pre-allocated RBDs is stored in iwl->rxq->rx_free.
57 * The page is either stolen - transferred to the upper layer
58 * or reused - added immediately to the iwl->rxq->rx_free list.
59 * + When the page is stolen - the driver updates the matching queue's used
60 * count, detaches the RBD and transfers it to the queue used list.
61 * When there are two used RBDs - they are transferred to the allocator empty
64 * When there are another 6 used RBDs - they are transferred to the allocator
65 * empty list and the driver tries to claim the pre-allocated buffers and
66 * add them to iwl->rxq->rx_free. If it fails - it continues to claim them
68 * When there are 8+ buffers in the free list - either from allocation or from
69 * 8 reused unstolen pages - restock is called to update the FW and indexes.
71 * the allocator has initial pool in the size of num_queues*(8-2) - the
76 * detached from the iwl->rxq. The driver 'processed' index is updated.
77 * + If there are no allocated buffers in iwl->rxq->rx_free,
78 * the READ INDEX is not incremented and iwl->status(RX_STALLED) is set.
88 * iwl_pcie_rxq_restock() Moves available buffers from rx_free into Rx
89 * queue, updates firmware pointers, and updates
93 * -- enable interrupts --
94 * ISR - iwl_rx() Detach iwl_rx_mem_buffers from pool up to the
96 * Moves the packet buffer from queue to rx_used.
101 * RBD life-cycle:
104 * rxq.pool -> rxq.rx_used -> rxq.rx_free -> rxq.queue
108 * rxq.queue -> rxq.rx_used -> allocator.rbd_empty ->
109 * allocator.rbd_allocated -> rxq.rx_free -> rxq.queue
111 * rxq.queue -> rxq.rx_free -> rxq.queue
117 * iwl_rxq_space - Return number of free slots available in queue.
121 /* Make sure rx queue size is a power of 2 */
122 WARN_ON(rxq->queue_size & (rxq->queue_size - 1));
125 * There can be up to (RX_QUEUE_SIZE - 1) free slots, to avoid ambiguity
130 return (rxq->read - rxq->write - 1) & (rxq->queue_size - 1);
134 * iwl_dma_addr2rbd_ptr - convert a DMA address to a uCode read buffer ptr
142 * iwl_pcie_rx_stop - stops the Rx DMA
146 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
151 } else if (trans->trans_cfg->mq_rx_supported) {
164 * iwl_pcie_rxq_inc_wr_ptr - Update the write pointer for the RX queue
171 lockdep_assert_held(&rxq->lock);
178 if (!trans->trans_cfg->base_params->shadow_reg_enable &&
179 test_bit(STATUS_TPOWER_PMI, &trans->status)) {
183 IWL_DEBUG_INFO(trans, "Rx queue requesting wakeup, GP1 = 0x%x\n",
187 rxq->need_update = true;
192 rxq->write_actual = round_down(rxq->write, 8);
193 if (!trans->trans_cfg->mq_rx_supported)
194 iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
195 else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
196 iwl_write32(trans, HBUS_TARG_WRPTR, rxq->write_actual |
197 HBUS_TARG_WRPTR_RX_Q(rxq->id));
199 iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id),
200 rxq->write_actual);
208 for (i = 0; i < trans->num_rx_queues; i++) {
209 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
211 if (!rxq->need_update)
213 spin_lock_bh(&rxq->lock);
215 rxq->need_update = false;
216 spin_unlock_bh(&rxq->lock);
224 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
225 struct iwl_rx_transfer_desc *bd = rxq->bd;
229 bd[rxq->write].addr = cpu_to_le64(rxb->page_dma);
230 bd[rxq->write].rbid = cpu_to_le16(rxb->vid);
232 __le64 *bd = rxq->bd;
234 bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid);
238 IWL_DEBUG_RX(trans, "Assigned virtual RB ID %u to queue %d index %d\n",
240 IWL_DEBUG_PCI_RW(trans, "Assigned virtual RB ID %u to queue %d index %d\n",
241 (u32)rxb->vid, rxq->id, rxq->write);
246 * iwl_pcie_rxmq_restock - restock implementation for multi-queue rx
255 * If the device isn't enabled - no need to try to add buffers...
262 if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
265 spin_lock_bh(&rxq->lock);
266 while (rxq->free_count) {
267 /* Get next free Rx buffer, remove from free list */
268 rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer,
270 list_del(&rxb->list);
271 rxb->invalid = false;
273 WARN_ON(rxb->page_dma & trans_pcie->supported_dma_mask);
274 /* Point to Rx buffer via next RBD in circular buffer */
276 rxq->write = (rxq->write + 1) & (rxq->queue_size - 1);
277 rxq->free_count--;
279 spin_unlock_bh(&rxq->lock);
285 if (rxq->write_actual != (rxq->write & ~0x7)) {
286 spin_lock_bh(&rxq->lock);
288 spin_unlock_bh(&rxq->lock);
293 * iwl_pcie_rxsq_restock - restock implementation for single queue rx
301 * If the device isn't enabled - not need to try to add buffers...
308 if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
311 spin_lock_bh(&rxq->lock);
312 while ((iwl_rxq_space(rxq) > 0) && (rxq->free_count)) {
313 __le32 *bd = (__le32 *)rxq->bd;
315 rxb = rxq->queue[rxq->write];
316 BUG_ON(rxb && rxb->page);
318 /* Get next free Rx buffer, remove from free list */
319 rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer,
321 list_del(&rxb->list);
322 rxb->invalid = false;
324 /* Point to Rx buffer via next RBD in circular buffer */
325 bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma);
326 rxq->queue[rxq->write] = rxb;
327 rxq->write = (rxq->write + 1) & RX_QUEUE_MASK;
328 rxq->free_count--;
330 spin_unlock_bh(&rxq->lock);
334 if (rxq->write_actual != (rxq->write & ~0x7)) {
335 spin_lock_bh(&rxq->lock);
337 spin_unlock_bh(&rxq->lock);
342 * iwl_pcie_rxq_restock - refill RX queue from pre-allocated pool
344 * If there are slots in the RX queue that need to be restocked,
345 * and we have free pre-allocated buffers, fill the ranks as much
355 if (trans->trans_cfg->mq_rx_supported)
362 * iwl_pcie_rx_alloc_page - allocates and returns a page.
369 unsigned int rbsize = iwl_trans_get_rb_size(trans_pcie->rx_buf_size);
370 unsigned int allocsize = PAGE_SIZE << trans_pcie->rx_page_order;
374 if (trans_pcie->rx_page_order > 0)
377 if (trans_pcie->alloc_page) {
378 spin_lock_bh(&trans_pcie->alloc_page_lock);
380 if (trans_pcie->alloc_page) {
381 *offset = trans_pcie->alloc_page_used;
382 page = trans_pcie->alloc_page;
383 trans_pcie->alloc_page_used += rbsize;
384 if (trans_pcie->alloc_page_used >= allocsize)
385 trans_pcie->alloc_page = NULL;
388 spin_unlock_bh(&trans_pcie->alloc_page_lock);
391 spin_unlock_bh(&trans_pcie->alloc_page_lock);
395 page = alloc_pages(gfp_mask, trans_pcie->rx_page_order);
399 trans_pcie->rx_page_order);
401 * Issue an error if we don't have enough pre-allocated
411 spin_lock_bh(&trans_pcie->alloc_page_lock);
412 if (!trans_pcie->alloc_page) {
414 trans_pcie->alloc_page = page;
415 trans_pcie->alloc_page_used = rbsize;
417 spin_unlock_bh(&trans_pcie->alloc_page_lock);
425 * iwl_pcie_rxq_alloc_rbs - allocate a page for each used RBD
427 * A used RBD is an Rx buffer that has been given to the stack. To use it again
443 spin_lock_bh(&rxq->lock);
444 if (list_empty(&rxq->rx_used)) {
445 spin_unlock_bh(&rxq->lock);
448 spin_unlock_bh(&rxq->lock);
454 spin_lock_bh(&rxq->lock);
456 if (list_empty(&rxq->rx_used)) {
457 spin_unlock_bh(&rxq->lock);
458 __free_pages(page, trans_pcie->rx_page_order);
461 rxb = list_first_entry(&rxq->rx_used, struct iwl_rx_mem_buffer,
463 list_del(&rxb->list);
464 spin_unlock_bh(&rxq->lock);
466 BUG_ON(rxb->page);
467 rxb->page = page;
468 rxb->offset = offset;
470 rxb->page_dma =
471 dma_map_page(trans->dev, page, rxb->offset,
472 trans_pcie->rx_buf_bytes,
474 if (dma_mapping_error(trans->dev, rxb->page_dma)) {
475 rxb->page = NULL;
476 spin_lock_bh(&rxq->lock);
477 list_add(&rxb->list, &rxq->rx_used);
478 spin_unlock_bh(&rxq->lock);
479 __free_pages(page, trans_pcie->rx_page_order);
483 spin_lock_bh(&rxq->lock);
485 list_add_tail(&rxb->list, &rxq->rx_free);
486 rxq->free_count++;
488 spin_unlock_bh(&rxq->lock);
497 if (!trans_pcie->rx_pool)
500 for (i = 0; i < RX_POOL_SIZE(trans_pcie->num_rx_bufs); i++) {
501 if (!trans_pcie->rx_pool[i].page)
503 dma_unmap_page(trans->dev, trans_pcie->rx_pool[i].page_dma,
504 trans_pcie->rx_buf_bytes, DMA_FROM_DEVICE);
505 __free_pages(trans_pcie->rx_pool[i].page,
506 trans_pcie->rx_page_order);
507 trans_pcie->rx_pool[i].page = NULL;
512 * iwl_pcie_rx_allocator - Allocates pages in the background for RX queues
520 struct iwl_rb_allocator *rba = &trans_pcie->rba;
522 int pending = atomic_read(&rba->req_pending);
526 /* If we were scheduled - there is at least one request */
527 spin_lock_bh(&rba->lock);
528 /* swap out the rba->rbd_empty to a local list */
529 list_replace_init(&rba->rbd_empty, &local_empty);
530 spin_unlock_bh(&rba->lock);
545 /* List should never be empty - each reused RBD is
554 BUG_ON(rxb->page);
557 page = iwl_pcie_rx_alloc_page(trans, &rxb->offset,
561 rxb->page = page;
564 rxb->page_dma = dma_map_page(trans->dev, page,
565 rxb->offset,
566 trans_pcie->rx_buf_bytes,
568 if (dma_mapping_error(trans->dev, rxb->page_dma)) {
569 rxb->page = NULL;
570 __free_pages(page, trans_pcie->rx_page_order);
575 list_move(&rxb->list, &local_allocated);
579 atomic_dec(&rba->req_pending);
580 pending--;
583 pending = atomic_read(&rba->req_pending);
590 spin_lock_bh(&rba->lock);
592 list_splice_tail(&local_allocated, &rba->rbd_allocated);
594 list_splice_tail_init(&rba->rbd_empty, &local_empty);
595 spin_unlock_bh(&rba->lock);
597 atomic_inc(&rba->req_ready);
601 spin_lock_bh(&rba->lock);
603 list_splice_tail(&local_empty, &rba->rbd_empty);
604 spin_unlock_bh(&rba->lock);
610 * iwl_pcie_rx_allocator_get - returns the pre-allocated pages
612 .* Called by queue when the queue posted allocation request and
614 * This function directly moves the allocated RBs to the queue's ownership
621 struct iwl_rb_allocator *rba = &trans_pcie->rba;
624 lockdep_assert_held(&rxq->lock);
627 * atomic_dec_if_positive returns req_ready - 1 for any scenario.
628 * If req_ready is 0 atomic_dec_if_positive will return -1 and this
631 * req_ready > 0, i.e. - there are ready requests and the function
634 if (atomic_dec_if_positive(&rba->req_ready) < 0)
637 spin_lock(&rba->lock);
639 /* Get next free Rx buffer, remove it from free list */
641 list_first_entry(&rba->rbd_allocated,
644 list_move(&rxb->list, &rxq->rx_free);
646 spin_unlock(&rba->lock);
648 rxq->used_count -= RX_CLAIM_REQ_ALLOC;
649 rxq->free_count += RX_CLAIM_REQ_ALLOC;
659 iwl_pcie_rx_allocator(trans_pcie->trans);
664 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
667 return trans->trans_cfg->mq_rx_supported ?
673 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
676 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
687 if (rxq->bd)
688 dma_free_coherent(trans->dev,
689 free_size * rxq->queue_size,
690 rxq->bd, rxq->bd_dma);
691 rxq->bd_dma = 0;
692 rxq->bd = NULL;
694 rxq->rb_stts_dma = 0;
695 rxq->rb_stts = NULL;
697 if (rxq->used_bd)
698 dma_free_coherent(trans->dev,
700 rxq->queue_size,
701 rxq->used_bd, rxq->used_bd_dma);
702 rxq->used_bd_dma = 0;
703 rxq->used_bd = NULL;
708 bool use_rx_td = (trans->trans_cfg->device_family >=
722 struct device *dev = trans->dev;
726 spin_lock_init(&rxq->lock);
727 if (trans->trans_cfg->mq_rx_supported)
728 rxq->queue_size = trans->cfg->num_rbds;
730 rxq->queue_size = RX_QUEUE_SIZE;
738 rxq->bd = dma_alloc_coherent(dev, free_size * rxq->queue_size,
739 &rxq->bd_dma, GFP_KERNEL);
740 if (!rxq->bd)
743 if (trans->trans_cfg->mq_rx_supported) {
744 rxq->used_bd = dma_alloc_coherent(dev,
746 rxq->queue_size,
747 &rxq->used_bd_dma,
749 if (!rxq->used_bd)
753 rxq->rb_stts = (u8 *)trans_pcie->base_rb_stts + rxq->id * rb_stts_size;
754 rxq->rb_stts_dma =
755 trans_pcie->base_rb_stts_dma + rxq->id * rb_stts_size;
760 for (i = 0; i < trans->num_rx_queues; i++) {
761 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
766 return -ENOMEM;
773 struct iwl_rb_allocator *rba = &trans_pcie->rba;
776 if (WARN_ON(trans_pcie->rxq))
777 return -EINVAL;
779 trans_pcie->rxq = kcalloc(trans->num_rx_queues, sizeof(struct iwl_rxq),
781 trans_pcie->rx_pool = kcalloc(RX_POOL_SIZE(trans_pcie->num_rx_bufs),
782 sizeof(trans_pcie->rx_pool[0]),
784 trans_pcie->global_table =
785 kcalloc(RX_POOL_SIZE(trans_pcie->num_rx_bufs),
786 sizeof(trans_pcie->global_table[0]),
788 if (!trans_pcie->rxq || !trans_pcie->rx_pool ||
789 !trans_pcie->global_table) {
790 ret = -ENOMEM;
794 spin_lock_init(&rba->lock);
800 trans_pcie->base_rb_stts =
801 dma_alloc_coherent(trans->dev,
802 rb_stts_size * trans->num_rx_queues,
803 &trans_pcie->base_rb_stts_dma,
805 if (!trans_pcie->base_rb_stts) {
806 ret = -ENOMEM;
810 for (i = 0; i < trans->num_rx_queues; i++) {
811 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
813 rxq->id = i;
821 if (trans_pcie->base_rb_stts) {
822 dma_free_coherent(trans->dev,
823 rb_stts_size * trans->num_rx_queues,
824 trans_pcie->base_rb_stts,
825 trans_pcie->base_rb_stts_dma);
826 trans_pcie->base_rb_stts = NULL;
827 trans_pcie->base_rb_stts_dma = 0;
829 kfree(trans_pcie->rx_pool);
830 trans_pcie->rx_pool = NULL;
831 kfree(trans_pcie->global_table);
832 trans_pcie->global_table = NULL;
833 kfree(trans_pcie->rxq);
834 trans_pcie->rxq = NULL;
845 switch (trans_pcie->rx_buf_size) {
863 /* Stop Rx DMA */
870 /* Reset driver's Rx queue write index */
875 (u32)(rxq->bd_dma >> 8));
877 /* Tell device where in DRAM to update its Rx status */
879 rxq->rb_stts_dma >> 4);
881 /* Enable Rx DMA
883 * the credit mechanism in 5000 HW RX FIFO
884 * Direct rx interrupts to hosts
885 * Rx buffer size 4 or 8k or 12k
903 if (trans->cfg->host_interrupt_operation_mode)
913 switch (trans_pcie->rx_buf_size) {
934 /* Stop Rx DMA */
936 /* disable free amd used rx queue operation */
939 for (i = 0; i < trans->num_rx_queues; i++) {
943 trans_pcie->rxq[i].bd_dma);
947 trans_pcie->rxq[i].used_bd_dma);
948 /* Tell device where in DRAM to update its Rx status */
951 trans_pcie->rxq[i].rb_stts_dma);
961 * Enable Rx DMA
962 * Rx buffer size 4 or 8k or 12k
975 * Set RX DMA chunk size to 64B for IOSF and 128B for PCIe
976 * Default queue is 0
983 trans->trans_cfg->integrated ?
986 /* Enable the relevant rx queues */
997 lockdep_assert_held(&rxq->lock);
999 INIT_LIST_HEAD(&rxq->rx_free);
1000 INIT_LIST_HEAD(&rxq->rx_used);
1001 rxq->free_count = 0;
1002 rxq->used_count = 0;
1005 static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget);
1019 trans_pcie = iwl_netdev_to_trans_pcie(napi->dev);
1020 trans = trans_pcie->trans;
1022 ret = iwl_pcie_rx_handle(trans, rxq->id, budget);
1025 rxq->id, ret, budget);
1028 spin_lock(&trans_pcie->irq_lock);
1029 if (test_bit(STATUS_INT_ENABLED, &trans->status))
1031 spin_unlock(&trans_pcie->irq_lock);
1033 napi_complete_done(&rxq->napi, ret);
1046 trans_pcie = iwl_netdev_to_trans_pcie(napi->dev);
1047 trans = trans_pcie->trans;
1049 ret = iwl_pcie_rx_handle(trans, rxq->id, budget);
1050 IWL_DEBUG_ISR(trans, "[%d] handled %d, budget %d\n", rxq->id, ret,
1054 int irq_line = rxq->id;
1057 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS &&
1058 rxq->id == 1)
1061 spin_lock(&trans_pcie->irq_lock);
1063 spin_unlock(&trans_pcie->irq_lock);
1065 napi_complete_done(&rxq->napi, ret);
1076 if (unlikely(!trans_pcie->rxq))
1079 for (i = 0; i < trans->num_rx_queues; i++) {
1080 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
1082 if (rxq && rxq->napi.poll)
1083 napi_synchronize(&rxq->napi);
1091 struct iwl_rb_allocator *rba = &trans_pcie->rba;
1094 if (!trans_pcie->rxq) {
1099 def_rxq = trans_pcie->rxq;
1101 cancel_work_sync(&rba->rx_alloc);
1103 spin_lock_bh(&rba->lock);
1104 atomic_set(&rba->req_pending, 0);
1105 atomic_set(&rba->req_ready, 0);
1106 INIT_LIST_HEAD(&rba->rbd_allocated);
1107 INIT_LIST_HEAD(&rba->rbd_empty);
1108 spin_unlock_bh(&rba->lock);
1110 /* free all first - we overwrite everything here */
1114 def_rxq->queue[i] = NULL;
1116 for (i = 0; i < trans->num_rx_queues; i++) {
1117 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
1119 spin_lock_bh(&rxq->lock);
1122 * and used all buffers, but have not restocked the Rx queue
1125 rxq->read = 0;
1126 rxq->write = 0;
1127 rxq->write_actual = 0;
1128 memset(rxq->rb_stts, 0,
1129 (trans->trans_cfg->device_family >=
1135 spin_unlock_bh(&rxq->lock);
1137 if (!rxq->napi.poll) {
1140 if (trans_pcie->msix_enabled)
1143 netif_napi_add(trans_pcie->napi_dev, &rxq->napi,
1145 napi_enable(&rxq->napi);
1150 /* move the pool to the default queue and allocator ownerships */
1151 queue_size = trans->trans_cfg->mq_rx_supported ?
1152 trans_pcie->num_rx_bufs - 1 : RX_QUEUE_SIZE;
1153 allocator_pool_size = trans->num_rx_queues *
1154 (RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC);
1158 struct iwl_rx_mem_buffer *rxb = &trans_pcie->rx_pool[i];
1161 list_add(&rxb->list, &rba->rbd_empty);
1163 list_add(&rxb->list, &def_rxq->rx_used);
1164 trans_pcie->global_table[i] = rxb;
1165 rxb->vid = (u16)(i + 1);
1166 rxb->invalid = true;
1182 if (trans->trans_cfg->mq_rx_supported)
1185 iwl_pcie_rx_hw_init(trans, trans_pcie->rxq);
1187 iwl_pcie_rxq_restock(trans, trans_pcie->rxq);
1189 spin_lock_bh(&trans_pcie->rxq->lock);
1190 iwl_pcie_rxq_inc_wr_ptr(trans, trans_pcie->rxq);
1191 spin_unlock_bh(&trans_pcie->rxq->lock);
1212 struct iwl_rb_allocator *rba = &trans_pcie->rba;
1219 if (!trans_pcie->rxq) {
1220 IWL_DEBUG_INFO(trans, "Free NULL rx context\n");
1224 cancel_work_sync(&rba->rx_alloc);
1228 if (trans_pcie->base_rb_stts) {
1229 dma_free_coherent(trans->dev,
1230 rb_stts_size * trans->num_rx_queues,
1231 trans_pcie->base_rb_stts,
1232 trans_pcie->base_rb_stts_dma);
1233 trans_pcie->base_rb_stts = NULL;
1234 trans_pcie->base_rb_stts_dma = 0;
1237 for (i = 0; i < trans->num_rx_queues; i++) {
1238 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
1242 if (rxq->napi.poll) {
1243 napi_disable(&rxq->napi);
1244 netif_napi_del(&rxq->napi);
1247 kfree(trans_pcie->rx_pool);
1248 kfree(trans_pcie->global_table);
1249 kfree(trans_pcie->rxq);
1251 if (trans_pcie->alloc_page)
1252 __free_pages(trans_pcie->alloc_page, trans_pcie->rx_page_order);
1258 spin_lock(&rba->lock);
1259 list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty);
1260 spin_unlock(&rba->lock);
1264 * iwl_pcie_rx_reuse_rbd - Recycle used RBDs
1267 * When there are 2 empty RBDs - a request for allocation is posted
1274 struct iwl_rb_allocator *rba = &trans_pcie->rba;
1278 list_add_tail(&rxb->list, &rxq->rx_used);
1284 rxq->used_count++;
1286 /* If we have RX_POST_REQ_ALLOC new released rx buffers -
1291 if ((rxq->used_count % RX_CLAIM_REQ_ALLOC) == RX_POST_REQ_ALLOC) {
1296 atomic_inc(&rba->req_pending);
1297 queue_work(rba->alloc_wq, &rba->rx_alloc);
1308 struct iwl_txq *txq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
1310 int max_len = trans_pcie->rx_buf_bytes;
1316 dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE);
1323 ._offset = rxb->offset + offset,
1324 ._rx_page_order = trans_pcie->rx_page_order,
1325 ._page = rxb->page,
1332 if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) {
1335 rxq->id, offset);
1339 WARN((le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >>
1340 FH_RSCSR_RXQ_POS != rxq->id,
1341 "frame on invalid queue - is on %d and indicates %d\n",
1342 rxq->id,
1343 (le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >>
1348 rxq->id, offset,
1350 WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)),
1351 pkt->hdr.group_id, pkt->hdr.cmd,
1352 le16_to_cpu(pkt->hdr.sequence));
1366 * to a (driver-originated) command.
1367 * If the packet (e.g. Rx frame) originated from uCode,
1369 * Ucode should set SEQ_RX_FRAME bit if ucode-originated,
1371 reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME);
1372 if (reclaim && !pkt->hdr.group_id) {
1375 for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) {
1376 if (trans_pcie->no_reclaim_cmds[i] ==
1377 pkt->hdr.cmd) {
1384 if (rxq->id == IWL_DEFAULT_RX_QUEUE)
1385 iwl_op_mode_rx(trans->op_mode, &rxq->napi,
1388 iwl_op_mode_rx_rss(trans->op_mode, &rxq->napi,
1389 &rxcb, rxq->id);
1397 u16 sequence = le16_to_cpu(pkt->hdr.sequence);
1401 kfree_sensitive(txq->entries[cmd_index].free_buf);
1402 txq->entries[cmd_index].free_buf = NULL;
1407 * as we reclaim the driver command queue */
1415 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
1419 /* page was stolen from us -- free our reference */
1421 __free_pages(rxb->page, trans_pcie->rx_page_order);
1422 rxb->page = NULL;
1426 * SKBs that fail to Rx correctly, add them back into the
1428 if (rxb->page != NULL) {
1429 rxb->page_dma =
1430 dma_map_page(trans->dev, rxb->page, rxb->offset,
1431 trans_pcie->rx_buf_bytes,
1433 if (dma_mapping_error(trans->dev, rxb->page_dma)) {
1439 __free_pages(rxb->page, trans_pcie->rx_page_order);
1440 rxb->page = NULL;
1443 list_add_tail(&rxb->list, &rxq->rx_free);
1444 rxq->free_count++;
1461 if (!trans->trans_cfg->mq_rx_supported) {
1462 rxb = rxq->queue[i];
1463 rxq->queue[i] = NULL;
1467 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
1468 struct iwl_rx_completion_desc_bz *cd = rxq->used_bd;
1472 } else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
1473 struct iwl_rx_completion_desc *cd = rxq->used_bd;
1478 __le32 *cd = rxq->used_bd;
1480 vid = le32_to_cpu(cd[i]) & 0x0FFF; /* 12-bit VID */
1483 if (!vid || vid > RX_POOL_SIZE(trans_pcie->num_rx_bufs))
1486 rxb = trans_pcie->global_table[vid - 1];
1487 if (rxb->invalid)
1490 IWL_DEBUG_RX(trans, "Got virtual RB ID %u\n", (u32)rxb->vid);
1492 rxb->invalid = true;
1503 * iwl_pcie_rx_handle - Main entry function for receiving responses from fw
1505 static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget)
1512 if (WARN_ON_ONCE(!trans_pcie->rxq || !trans_pcie->rxq[queue].bd))
1515 rxq = &trans_pcie->rxq[queue];
1518 spin_lock(&rxq->lock);
1519 /* uCode's read index (stored in shared DRAM) indicates the last Rx
1522 i = rxq->read;
1524 /* W/A 9000 device step A0 wrap-around bug */
1525 r &= (rxq->queue_size - 1);
1527 /* Rx interrupt, but nothing sent from uCode */
1529 IWL_DEBUG_RX(trans, "Q %d: HW = SW = %d\n", rxq->id, r);
1532 struct iwl_rb_allocator *rba = &trans_pcie->rba;
1536 atomic_read(&trans_pcie->rba.req_pending) *
1540 if (unlikely(rb_pending_alloc >= rxq->queue_size / 2 &&
1545 "RX path is in emergency. Pending allocations %d\n",
1549 IWL_DEBUG_RX(trans, "Q %d: HW = %d, SW = %d\n", rxq->id, r, i);
1555 if (unlikely(join || rxq->next_rb_is_fragment)) {
1556 rxq->next_rb_is_fragment = join;
1558 * We can only get a multi-RB in the following cases:
1559 * - firmware issue, sending a too big notification
1560 * - sniffer mode with a large A-MSDU
1561 * - large MTU frames (>2k)
1562 * since the multi-RB functionality is limited to newer
1569 list_add_tail(&rxb->list, &rxq->rx_free);
1570 rxq->free_count++;
1575 i = (i + 1) & (rxq->queue_size - 1);
1578 * If we have RX_CLAIM_REQ_ALLOC released rx buffers -
1579 * try to claim the pre-allocated buffers from the allocator.
1580 * If not ready - will try to reclaim next time.
1581 * There is no need to reschedule work - allocator exits only
1584 if (rxq->used_count >= RX_CLAIM_REQ_ALLOC)
1587 if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 && !emergency) {
1594 if (rb_pending_alloc < rxq->queue_size / 3) {
1596 "RX path exited emergency. Pending allocations %d\n",
1601 rxq->read = i;
1602 spin_unlock(&rxq->lock);
1611 rxq->read = i;
1612 spin_unlock(&rxq->lock);
1616 * those RBDs are in the used list, but are not tracked by the queue's
1622 * by the queue.
1623 * by allocating them here, they are now in the queue free list, and
1636 u8 queue = entry->entry;
1637 struct msix_entry *entries = entry - queue;
1643 * iwl_pcie_rx_msix_handle - Main entry function for receiving responses from fw
1644 * This interrupt handler should be used with RSS queue only.
1650 struct iwl_trans *trans = trans_pcie->trans;
1653 trace_iwlwifi_dev_irq_msix(trans->dev, entry, false, 0, 0);
1655 if (WARN_ON(entry->entry >= trans->num_rx_queues))
1658 if (!trans_pcie->rxq) {
1661 "[%d] Got MSI-X interrupt before we have Rx queues\n",
1662 entry->entry);
1666 rxq = &trans_pcie->rxq[entry->entry];
1667 lock_map_acquire(&trans->sync_cmd_lockdep_map);
1668 IWL_DEBUG_ISR(trans, "[%d] Got interrupt\n", entry->entry);
1671 if (!napi_schedule(&rxq->napi))
1672 iwl_pcie_clear_irq(trans, entry->entry);
1675 lock_map_release(&trans->sync_cmd_lockdep_map);
1681 * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card
1689 if (trans->cfg->internal_wimax_coex &&
1690 !trans->cfg->apmg_not_supported &&
1695 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1696 iwl_op_mode_wimax_active(trans->op_mode);
1697 wake_up(&trans->wait_command_queue);
1701 for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
1702 if (!trans_pcie->txqs.txq[i])
1704 del_timer(&trans_pcie->txqs.txq[i]->stuck_timer);
1711 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1712 wake_up(&trans->wait_command_queue);
1719 lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock);
1721 trace_iwlwifi_dev_irq(trans->dev);
1726 /* the thread will service interrupts and re-enable them */
1730 /* a device (PCI-E) page is 4096 bytes long */
1750 trace_iwlwifi_dev_irq(trans->dev);
1755 read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]);
1756 trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read);
1767 trans_pcie->ict_index, read);
1768 trans_pcie->ict_tbl[trans_pcie->ict_index] = 0;
1769 trans_pcie->ict_index =
1770 ((trans_pcie->ict_index + 1) & (ICT_COUNT - 1));
1772 read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]);
1773 trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index,
1782 * this is a w/a for a h/w bug. the h/w bug may cause the Rx bit
1785 * so we use them to decide on the real state of the Rx bit.
1798 struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
1801 mutex_lock(&trans_pcie->mutex);
1802 prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1805 set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1806 set_bit(STATUS_RFKILL_HW, &trans->status);
1808 if (trans_pcie->opmode_down)
1811 report = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1816 isr_stats->rfkill++;
1820 mutex_unlock(&trans_pcie->mutex);
1824 &trans->status))
1827 wake_up(&trans->wait_command_queue);
1829 clear_bit(STATUS_RFKILL_HW, &trans->status);
1830 if (trans_pcie->opmode_down)
1831 clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1839 struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
1844 lock_map_acquire(&trans->sync_cmd_lockdep_map);
1846 spin_lock_bh(&trans_pcie->irq_lock);
1851 if (likely(trans_pcie->use_ict))
1860 inta, trans_pcie->inta_mask,
1863 if (inta & (~trans_pcie->inta_mask))
1866 inta & (~trans_pcie->inta_mask));
1870 inta &= trans_pcie->inta_mask;
1880 * Re-enable interrupts here since we don't
1883 if (test_bit(STATUS_INT_ENABLED, &trans->status))
1885 spin_unlock_bh(&trans_pcie->irq_lock);
1886 lock_map_release(&trans->sync_cmd_lockdep_map);
1896 spin_unlock_bh(&trans_pcie->irq_lock);
1911 iwl_write32(trans, CSR_INT, inta | ~trans_pcie->inta_mask);
1919 spin_unlock_bh(&trans_pcie->irq_lock);
1928 isr_stats->hw++;
1940 isr_stats->sch++;
1943 /* Alive notification via Rx interrupt will do the real work */
1946 isr_stats->alive++;
1947 if (trans->trans_cfg->gen2) {
1952 iwl_pcie_rxmq_restock(trans, trans_pcie->rxq);
1970 isr_stats->ctkill++;
1978 isr_stats->sw++;
1983 /* uCode wakes up after power-down sleep */
1989 isr_stats->wakeup++;
1995 * Rx "responses" (frame-received notification), and other
1999 IWL_DEBUG_ISR(trans, "Rx interrupt\n");
2010 /* Sending RX interrupt require many steps to be done in the
2012 * 1- write interrupt to current index in ICT table.
2013 * 2- dma RX frame.
2014 * 3- update RX shared data to indicate last write index.
2015 * 4- send interrupt.
2016 * This could lead to RX race, driver could receive RX interrupt
2018 * periodic interrupt will detect any dangling Rx activity.
2021 /* Disable periodic interrupt; we use it as just a one-shot. */
2027 * real RX interrupt (instead of just periodic int), to catch
2028 * any dangling Rx interrupt. If it was just the periodic
2029 * interrupt, there was no dangling Rx activity, and no need
2030 * to extend the periodic interrupt; one-shot is enough.
2036 isr_stats->rx++;
2039 if (napi_schedule_prep(&trans_pcie->rxq[0].napi)) {
2041 __napi_schedule(&trans_pcie->rxq[0].napi);
2050 isr_stats->tx++;
2053 trans_pcie->ucode_write_complete = true;
2054 wake_up(&trans_pcie->ucode_write_waitq);
2056 if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
2057 trans_pcie->imr_status = IMR_D2S_COMPLETED;
2058 wake_up(&trans_pcie->ucode_write_waitq);
2064 isr_stats->unhandled++;
2067 if (inta & ~(trans_pcie->inta_mask)) {
2069 inta & ~trans_pcie->inta_mask);
2073 spin_lock_bh(&trans_pcie->irq_lock);
2074 /* only Re-enable all interrupt if disabled by irq */
2075 if (test_bit(STATUS_INT_ENABLED, &trans->status))
2080 /* Re-enable RF_KILL if it occurred */
2083 /* Re-enable the ALIVE / Rx interrupt if it occurred */
2086 spin_unlock_bh(&trans_pcie->irq_lock);
2090 lock_map_release(&trans->sync_cmd_lockdep_map);
2105 if (trans_pcie->ict_tbl) {
2106 dma_free_coherent(trans->dev, ICT_SIZE,
2107 trans_pcie->ict_tbl,
2108 trans_pcie->ict_tbl_dma);
2109 trans_pcie->ict_tbl = NULL;
2110 trans_pcie->ict_tbl_dma = 0;
2123 trans_pcie->ict_tbl =
2124 dma_alloc_coherent(trans->dev, ICT_SIZE,
2125 &trans_pcie->ict_tbl_dma, GFP_KERNEL);
2126 if (!trans_pcie->ict_tbl)
2127 return -ENOMEM;
2130 if (WARN_ON(trans_pcie->ict_tbl_dma & (ICT_SIZE - 1))) {
2132 return -EINVAL;
2146 if (!trans_pcie->ict_tbl)
2149 spin_lock_bh(&trans_pcie->irq_lock);
2152 memset(trans_pcie->ict_tbl, 0, ICT_SIZE);
2154 val = trans_pcie->ict_tbl_dma >> ICT_SHIFT;
2163 trans_pcie->use_ict = true;
2164 trans_pcie->ict_index = 0;
2165 iwl_write32(trans, CSR_INT, trans_pcie->inta_mask);
2167 spin_unlock_bh(&trans_pcie->irq_lock);
2175 spin_lock_bh(&trans_pcie->irq_lock);
2176 trans_pcie->use_ict = false;
2177 spin_unlock_bh(&trans_pcie->irq_lock);
2188 * back-to-back ISRs and sporadic interrupts from our NIC.
2189 * If we have something to service, the tasklet will re-enable ints.
2190 * If we *don't* have something, we'll re-enable before leaving here.
2206 struct iwl_trans *trans = trans_pcie->trans;
2207 struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2213 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
2216 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
2219 lock_map_acquire(&trans->sync_cmd_lockdep_map);
2221 spin_lock_bh(&trans_pcie->irq_lock);
2229 spin_unlock_bh(&trans_pcie->irq_lock);
2231 trace_iwlwifi_dev_irq_msix(trans->dev, entry, true, inta_fh, inta_hw);
2235 lock_map_release(&trans->sync_cmd_lockdep_map);
2243 entry->entry, inta_fh, trans_pcie->fh_mask,
2245 if (inta_fh & ~trans_pcie->fh_mask)
2248 inta_fh & ~trans_pcie->fh_mask);
2252 inta_fh &= trans_pcie->fh_mask;
2254 if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) &&
2257 if (napi_schedule_prep(&trans_pcie->rxq[0].napi)) {
2259 __napi_schedule(&trans_pcie->rxq[0].napi);
2264 if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) &&
2267 if (napi_schedule_prep(&trans_pcie->rxq[1].napi)) {
2269 __napi_schedule(&trans_pcie->rxq[1].napi);
2276 trans_pcie->imr_status == IMR_D2S_REQUESTED) {
2278 isr_stats->tx++;
2281 if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
2282 trans_pcie->imr_status = IMR_D2S_COMPLETED;
2283 wake_up(&trans_pcie->ucode_write_waitq);
2287 isr_stats->tx++;
2292 trans_pcie->ucode_write_complete = true;
2293 wake_up(&trans_pcie->ucode_write_waitq);
2296 if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
2297 trans_pcie->imr_status = IMR_D2S_COMPLETED;
2298 wake_up(&trans_pcie->ucode_write_waitq);
2302 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
2318 isr_stats->sw++;
2320 if (trans_pcie->imr_status == IMR_D2S_REQUESTED) {
2321 trans_pcie->imr_status = IMR_D2S_ERROR;
2322 wake_up(&trans_pcie->imr_waitq);
2323 } else if (trans_pcie->fw_reset_state == FW_RESET_REQUESTED) {
2324 trans_pcie->fw_reset_state = FW_RESET_ERROR;
2325 wake_up(&trans_pcie->fw_reset_waitq);
2336 entry->entry, inta_hw, trans_pcie->hw_mask,
2338 if (inta_hw & ~trans_pcie->hw_mask)
2341 inta_hw & ~trans_pcie->hw_mask);
2345 inta_hw &= trans_pcie->hw_mask;
2347 /* Alive notification via Rx interrupt will do the real work */
2350 isr_stats->alive++;
2351 if (trans->trans_cfg->gen2) {
2353 iwl_pcie_rxmq_restock(trans, trans_pcie->rxq);
2362 if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP && trans_pcie->prph_info) {
2364 le32_to_cpu(trans_pcie->prph_info->sleep_notif);
2370 trans_pcie->sx_complete = true;
2371 wake_up(&trans_pcie->sx_waitq);
2373 /* uCode wakes up after power-down sleep */
2378 isr_stats->wakeup++;
2385 isr_stats->ctkill++;
2396 isr_stats->hw++;
2397 trans->dbg.hw_error = true;
2403 trans_pcie->fw_reset_state = FW_RESET_OK;
2404 wake_up(&trans_pcie->fw_reset_waitq);
2408 iwl_pcie_clear_irq(trans, entry->entry);
2410 lock_map_release(&trans->sync_cmd_lockdep_map);