xref: /linux/drivers/net/wireless/intel/iwlwifi/pcie/rx.c (revision e5c86679d5e864947a52fb31e45a425dea3e7fa9)
1 /******************************************************************************
2  *
3  * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
4  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
5  * Copyright(c) 2016 Intel Deutschland GmbH
6  *
7  * Portions of this file are derived from the ipw3945 project, as well
8  * as portions of the ieee80211 subsystem header files.
9  *
10  * This program is free software; you can redistribute it and/or modify it
11  * under the terms of version 2 of the GNU General Public License as
12  * published by the Free Software Foundation.
13  *
14  * This program is distributed in the hope that it will be useful, but WITHOUT
15  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
17  * more details.
18  *
19  * You should have received a copy of the GNU General Public License along with
20  * this program; if not, write to the Free Software Foundation, Inc.,
21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
22  *
23  * The full GNU General Public License is included in this distribution in the
24  * file called LICENSE.
25  *
26  * Contact Information:
27  *  Intel Linux Wireless <linuxwifi@intel.com>
28  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
29  *
30  *****************************************************************************/
31 #include <linux/sched.h>
32 #include <linux/wait.h>
33 #include <linux/gfp.h>
34 
35 #include "iwl-prph.h"
36 #include "iwl-io.h"
37 #include "internal.h"
38 #include "iwl-op-mode.h"
39 
40 /******************************************************************************
41  *
42  * RX path functions
43  *
44  ******************************************************************************/
45 
46 /*
47  * Rx theory of operation
48  *
49  * Driver allocates a circular buffer of Receive Buffer Descriptors (RBDs),
50  * each of which point to Receive Buffers to be filled by the NIC.  These get
51  * used not only for Rx frames, but for any command response or notification
52  * from the NIC.  The driver and NIC manage the Rx buffers by means
53  * of indexes into the circular buffer.
54  *
55  * Rx Queue Indexes
56  * The host/firmware share two index registers for managing the Rx buffers.
57  *
58  * The READ index maps to the first position that the firmware may be writing
59  * to -- the driver can read up to (but not including) this position and get
60  * good data.
61  * The READ index is managed by the firmware once the card is enabled.
62  *
63  * The WRITE index maps to the last position the driver has read from -- the
64  * position preceding WRITE is the last slot the firmware can place a packet.
65  *
66  * The queue is empty (no good data) if WRITE = READ - 1, and is full if
67  * WRITE = READ.
68  *
69  * During initialization, the host sets up the READ queue position to the first
70  * INDEX position, and WRITE to the last (READ - 1 wrapped)
71  *
72  * When the firmware places a packet in a buffer, it will advance the READ index
73  * and fire the RX interrupt.  The driver can then query the READ index and
74  * process as many packets as possible, moving the WRITE index forward as it
75  * resets the Rx queue buffers with new memory.
76  *
77  * The management in the driver is as follows:
78  * + A list of pre-allocated RBDs is stored in iwl->rxq->rx_free.
79  *   When the interrupt handler is called, the request is processed.
80  *   The page is either stolen - transferred to the upper layer
81  *   or reused - added immediately to the iwl->rxq->rx_free list.
82  * + When the page is stolen - the driver updates the matching queue's used
83  *   count, detaches the RBD and transfers it to the queue used list.
84  *   When there are two used RBDs - they are transferred to the allocator empty
85  *   list. Work is then scheduled for the allocator to start allocating
86  *   eight buffers.
87  *   When there are another 6 used RBDs - they are transferred to the allocator
88  *   empty list and the driver tries to claim the pre-allocated buffers and
89  *   add them to iwl->rxq->rx_free. If it fails - it continues to claim them
90  *   until ready.
91  *   When there are 8+ buffers in the free list - either from allocation or from
92  *   8 reused unstolen pages - restock is called to update the FW and indexes.
93  * + In order to make sure the allocator always has RBDs to use for allocation
94  *   the allocator has initial pool in the size of num_queues*(8-2) - the
95  *   maximum missing RBDs per allocation request (request posted with 2
96  *    empty RBDs, there is no guarantee when the other 6 RBDs are supplied).
97  *   The queues supplies the recycle of the rest of the RBDs.
98  * + A received packet is processed and handed to the kernel network stack,
99  *   detached from the iwl->rxq.  The driver 'processed' index is updated.
100  * + If there are no allocated buffers in iwl->rxq->rx_free,
101  *   the READ INDEX is not incremented and iwl->status(RX_STALLED) is set.
102  *   If there were enough free buffers and RX_STALLED is set it is cleared.
103  *
104  *
105  * Driver sequence:
106  *
107  * iwl_rxq_alloc()            Allocates rx_free
108  * iwl_pcie_rx_replenish()    Replenishes rx_free list from rx_used, and calls
109  *                            iwl_pcie_rxq_restock.
110  *                            Used only during initialization.
111  * iwl_pcie_rxq_restock()     Moves available buffers from rx_free into Rx
112  *                            queue, updates firmware pointers, and updates
113  *                            the WRITE index.
114  * iwl_pcie_rx_allocator()     Background work for allocating pages.
115  *
116  * -- enable interrupts --
117  * ISR - iwl_rx()             Detach iwl_rx_mem_buffers from pool up to the
118  *                            READ INDEX, detaching the SKB from the pool.
119  *                            Moves the packet buffer from queue to rx_used.
120  *                            Posts and claims requests to the allocator.
121  *                            Calls iwl_pcie_rxq_restock to refill any empty
122  *                            slots.
123  *
124  * RBD life-cycle:
125  *
126  * Init:
127  * rxq.pool -> rxq.rx_used -> rxq.rx_free -> rxq.queue
128  *
129  * Regular Receive interrupt:
130  * Page Stolen:
131  * rxq.queue -> rxq.rx_used -> allocator.rbd_empty ->
132  * allocator.rbd_allocated -> rxq.rx_free -> rxq.queue
133  * Page not Stolen:
134  * rxq.queue -> rxq.rx_free -> rxq.queue
135  * ...
136  *
137  */
138 
139 /*
140  * iwl_rxq_space - Return number of free slots available in queue.
141  */
142 static int iwl_rxq_space(const struct iwl_rxq *rxq)
143 {
144 	/* Make sure rx queue size is a power of 2 */
145 	WARN_ON(rxq->queue_size & (rxq->queue_size - 1));
146 
147 	/*
148 	 * There can be up to (RX_QUEUE_SIZE - 1) free slots, to avoid ambiguity
149 	 * between empty and completely full queues.
150 	 * The following is equivalent to modulo by RX_QUEUE_SIZE and is well
151 	 * defined for negative dividends.
152 	 */
153 	return (rxq->read - rxq->write - 1) & (rxq->queue_size - 1);
154 }
155 
156 /*
157  * iwl_dma_addr2rbd_ptr - convert a DMA address to a uCode read buffer ptr
158  */
159 static inline __le32 iwl_pcie_dma_addr2rbd_ptr(dma_addr_t dma_addr)
160 {
161 	return cpu_to_le32((u32)(dma_addr >> 8));
162 }
163 
164 /*
165  * iwl_pcie_rx_stop - stops the Rx DMA
166  */
167 int iwl_pcie_rx_stop(struct iwl_trans *trans)
168 {
169 	if (trans->cfg->mq_rx_supported) {
170 		iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0);
171 		return iwl_poll_prph_bit(trans, RFH_GEN_STATUS,
172 					   RXF_DMA_IDLE, RXF_DMA_IDLE, 1000);
173 	} else {
174 		iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
175 		return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG,
176 					   FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE,
177 					   1000);
178 	}
179 }
180 
181 /*
182  * iwl_pcie_rxq_inc_wr_ptr - Update the write pointer for the RX queue
183  */
184 static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans,
185 				    struct iwl_rxq *rxq)
186 {
187 	u32 reg;
188 
189 	lockdep_assert_held(&rxq->lock);
190 
191 	/*
192 	 * explicitly wake up the NIC if:
193 	 * 1. shadow registers aren't enabled
194 	 * 2. there is a chance that the NIC is asleep
195 	 */
196 	if (!trans->cfg->base_params->shadow_reg_enable &&
197 	    test_bit(STATUS_TPOWER_PMI, &trans->status)) {
198 		reg = iwl_read32(trans, CSR_UCODE_DRV_GP1);
199 
200 		if (reg & CSR_UCODE_DRV_GP1_BIT_MAC_SLEEP) {
201 			IWL_DEBUG_INFO(trans, "Rx queue requesting wakeup, GP1 = 0x%x\n",
202 				       reg);
203 			iwl_set_bit(trans, CSR_GP_CNTRL,
204 				    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
205 			rxq->need_update = true;
206 			return;
207 		}
208 	}
209 
210 	rxq->write_actual = round_down(rxq->write, 8);
211 	if (trans->cfg->mq_rx_supported)
212 		iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id),
213 			    rxq->write_actual);
214 	else
215 		iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
216 }
217 
218 static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans)
219 {
220 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
221 	int i;
222 
223 	for (i = 0; i < trans->num_rx_queues; i++) {
224 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
225 
226 		if (!rxq->need_update)
227 			continue;
228 		spin_lock(&rxq->lock);
229 		iwl_pcie_rxq_inc_wr_ptr(trans, rxq);
230 		rxq->need_update = false;
231 		spin_unlock(&rxq->lock);
232 	}
233 }
234 
235 /*
236  * iwl_pcie_rxmq_restock - restock implementation for multi-queue rx
237  */
238 static void iwl_pcie_rxmq_restock(struct iwl_trans *trans,
239 				  struct iwl_rxq *rxq)
240 {
241 	struct iwl_rx_mem_buffer *rxb;
242 
243 	/*
244 	 * If the device isn't enabled - no need to try to add buffers...
245 	 * This can happen when we stop the device and still have an interrupt
246 	 * pending. We stop the APM before we sync the interrupts because we
247 	 * have to (see comment there). On the other hand, since the APM is
248 	 * stopped, we cannot access the HW (in particular not prph).
249 	 * So don't try to restock if the APM has been already stopped.
250 	 */
251 	if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
252 		return;
253 
254 	spin_lock(&rxq->lock);
255 	while (rxq->free_count) {
256 		__le64 *bd = (__le64 *)rxq->bd;
257 
258 		/* Get next free Rx buffer, remove from free list */
259 		rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer,
260 				       list);
261 		list_del(&rxb->list);
262 		rxb->invalid = false;
263 		/* 12 first bits are expected to be empty */
264 		WARN_ON(rxb->page_dma & DMA_BIT_MASK(12));
265 		/* Point to Rx buffer via next RBD in circular buffer */
266 		bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid);
267 		rxq->write = (rxq->write + 1) & MQ_RX_TABLE_MASK;
268 		rxq->free_count--;
269 	}
270 	spin_unlock(&rxq->lock);
271 
272 	/*
273 	 * If we've added more space for the firmware to place data, tell it.
274 	 * Increment device's write pointer in multiples of 8.
275 	 */
276 	if (rxq->write_actual != (rxq->write & ~0x7)) {
277 		spin_lock(&rxq->lock);
278 		iwl_pcie_rxq_inc_wr_ptr(trans, rxq);
279 		spin_unlock(&rxq->lock);
280 	}
281 }
282 
283 /*
284  * iwl_pcie_rxsq_restock - restock implementation for single queue rx
285  */
286 static void iwl_pcie_rxsq_restock(struct iwl_trans *trans,
287 				  struct iwl_rxq *rxq)
288 {
289 	struct iwl_rx_mem_buffer *rxb;
290 
291 	/*
292 	 * If the device isn't enabled - not need to try to add buffers...
293 	 * This can happen when we stop the device and still have an interrupt
294 	 * pending. We stop the APM before we sync the interrupts because we
295 	 * have to (see comment there). On the other hand, since the APM is
296 	 * stopped, we cannot access the HW (in particular not prph).
297 	 * So don't try to restock if the APM has been already stopped.
298 	 */
299 	if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
300 		return;
301 
302 	spin_lock(&rxq->lock);
303 	while ((iwl_rxq_space(rxq) > 0) && (rxq->free_count)) {
304 		__le32 *bd = (__le32 *)rxq->bd;
305 		/* The overwritten rxb must be a used one */
306 		rxb = rxq->queue[rxq->write];
307 		BUG_ON(rxb && rxb->page);
308 
309 		/* Get next free Rx buffer, remove from free list */
310 		rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer,
311 				       list);
312 		list_del(&rxb->list);
313 		rxb->invalid = false;
314 
315 		/* Point to Rx buffer via next RBD in circular buffer */
316 		bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma);
317 		rxq->queue[rxq->write] = rxb;
318 		rxq->write = (rxq->write + 1) & RX_QUEUE_MASK;
319 		rxq->free_count--;
320 	}
321 	spin_unlock(&rxq->lock);
322 
323 	/* If we've added more space for the firmware to place data, tell it.
324 	 * Increment device's write pointer in multiples of 8. */
325 	if (rxq->write_actual != (rxq->write & ~0x7)) {
326 		spin_lock(&rxq->lock);
327 		iwl_pcie_rxq_inc_wr_ptr(trans, rxq);
328 		spin_unlock(&rxq->lock);
329 	}
330 }
331 
332 /*
333  * iwl_pcie_rxq_restock - refill RX queue from pre-allocated pool
334  *
335  * If there are slots in the RX queue that need to be restocked,
336  * and we have free pre-allocated buffers, fill the ranks as much
337  * as we can, pulling from rx_free.
338  *
339  * This moves the 'write' index forward to catch up with 'processed', and
340  * also updates the memory address in the firmware to reference the new
341  * target buffer.
342  */
343 static
344 void iwl_pcie_rxq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq)
345 {
346 	if (trans->cfg->mq_rx_supported)
347 		iwl_pcie_rxmq_restock(trans, rxq);
348 	else
349 		iwl_pcie_rxsq_restock(trans, rxq);
350 }
351 
352 /*
353  * iwl_pcie_rx_alloc_page - allocates and returns a page.
354  *
355  */
356 static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans,
357 					   gfp_t priority)
358 {
359 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
360 	struct page *page;
361 	gfp_t gfp_mask = priority;
362 
363 	if (trans_pcie->rx_page_order > 0)
364 		gfp_mask |= __GFP_COMP;
365 
366 	/* Alloc a new receive buffer */
367 	page = alloc_pages(gfp_mask, trans_pcie->rx_page_order);
368 	if (!page) {
369 		if (net_ratelimit())
370 			IWL_DEBUG_INFO(trans, "alloc_pages failed, order: %d\n",
371 				       trans_pcie->rx_page_order);
372 		/*
373 		 * Issue an error if we don't have enough pre-allocated
374 		  * buffers.
375 `		 */
376 		if (!(gfp_mask & __GFP_NOWARN) && net_ratelimit())
377 			IWL_CRIT(trans,
378 				 "Failed to alloc_pages\n");
379 		return NULL;
380 	}
381 	return page;
382 }
383 
384 /*
385  * iwl_pcie_rxq_alloc_rbs - allocate a page for each used RBD
386  *
387  * A used RBD is an Rx buffer that has been given to the stack. To use it again
388  * a page must be allocated and the RBD must point to the page. This function
389  * doesn't change the HW pointer but handles the list of pages that is used by
390  * iwl_pcie_rxq_restock. The latter function will update the HW to use the newly
391  * allocated buffers.
392  */
393 static void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority,
394 				   struct iwl_rxq *rxq)
395 {
396 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
397 	struct iwl_rx_mem_buffer *rxb;
398 	struct page *page;
399 
400 	while (1) {
401 		spin_lock(&rxq->lock);
402 		if (list_empty(&rxq->rx_used)) {
403 			spin_unlock(&rxq->lock);
404 			return;
405 		}
406 		spin_unlock(&rxq->lock);
407 
408 		/* Alloc a new receive buffer */
409 		page = iwl_pcie_rx_alloc_page(trans, priority);
410 		if (!page)
411 			return;
412 
413 		spin_lock(&rxq->lock);
414 
415 		if (list_empty(&rxq->rx_used)) {
416 			spin_unlock(&rxq->lock);
417 			__free_pages(page, trans_pcie->rx_page_order);
418 			return;
419 		}
420 		rxb = list_first_entry(&rxq->rx_used, struct iwl_rx_mem_buffer,
421 				       list);
422 		list_del(&rxb->list);
423 		spin_unlock(&rxq->lock);
424 
425 		BUG_ON(rxb->page);
426 		rxb->page = page;
427 		/* Get physical address of the RB */
428 		rxb->page_dma =
429 			dma_map_page(trans->dev, page, 0,
430 				     PAGE_SIZE << trans_pcie->rx_page_order,
431 				     DMA_FROM_DEVICE);
432 		if (dma_mapping_error(trans->dev, rxb->page_dma)) {
433 			rxb->page = NULL;
434 			spin_lock(&rxq->lock);
435 			list_add(&rxb->list, &rxq->rx_used);
436 			spin_unlock(&rxq->lock);
437 			__free_pages(page, trans_pcie->rx_page_order);
438 			return;
439 		}
440 
441 		spin_lock(&rxq->lock);
442 
443 		list_add_tail(&rxb->list, &rxq->rx_free);
444 		rxq->free_count++;
445 
446 		spin_unlock(&rxq->lock);
447 	}
448 }
449 
450 static void iwl_pcie_free_rbs_pool(struct iwl_trans *trans)
451 {
452 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
453 	int i;
454 
455 	for (i = 0; i < RX_POOL_SIZE; i++) {
456 		if (!trans_pcie->rx_pool[i].page)
457 			continue;
458 		dma_unmap_page(trans->dev, trans_pcie->rx_pool[i].page_dma,
459 			       PAGE_SIZE << trans_pcie->rx_page_order,
460 			       DMA_FROM_DEVICE);
461 		__free_pages(trans_pcie->rx_pool[i].page,
462 			     trans_pcie->rx_page_order);
463 		trans_pcie->rx_pool[i].page = NULL;
464 	}
465 }
466 
467 /*
468  * iwl_pcie_rx_allocator - Allocates pages in the background for RX queues
469  *
470  * Allocates for each received request 8 pages
471  * Called as a scheduled work item.
472  */
473 static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
474 {
475 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
476 	struct iwl_rb_allocator *rba = &trans_pcie->rba;
477 	struct list_head local_empty;
478 	int pending = atomic_xchg(&rba->req_pending, 0);
479 
480 	IWL_DEBUG_RX(trans, "Pending allocation requests = %d\n", pending);
481 
482 	/* If we were scheduled - there is at least one request */
483 	spin_lock(&rba->lock);
484 	/* swap out the rba->rbd_empty to a local list */
485 	list_replace_init(&rba->rbd_empty, &local_empty);
486 	spin_unlock(&rba->lock);
487 
488 	while (pending) {
489 		int i;
490 		LIST_HEAD(local_allocated);
491 		gfp_t gfp_mask = GFP_KERNEL;
492 
493 		/* Do not post a warning if there are only a few requests */
494 		if (pending < RX_PENDING_WATERMARK)
495 			gfp_mask |= __GFP_NOWARN;
496 
497 		for (i = 0; i < RX_CLAIM_REQ_ALLOC;) {
498 			struct iwl_rx_mem_buffer *rxb;
499 			struct page *page;
500 
501 			/* List should never be empty - each reused RBD is
502 			 * returned to the list, and initial pool covers any
503 			 * possible gap between the time the page is allocated
504 			 * to the time the RBD is added.
505 			 */
506 			BUG_ON(list_empty(&local_empty));
507 			/* Get the first rxb from the rbd list */
508 			rxb = list_first_entry(&local_empty,
509 					       struct iwl_rx_mem_buffer, list);
510 			BUG_ON(rxb->page);
511 
512 			/* Alloc a new receive buffer */
513 			page = iwl_pcie_rx_alloc_page(trans, gfp_mask);
514 			if (!page)
515 				continue;
516 			rxb->page = page;
517 
518 			/* Get physical address of the RB */
519 			rxb->page_dma = dma_map_page(trans->dev, page, 0,
520 					PAGE_SIZE << trans_pcie->rx_page_order,
521 					DMA_FROM_DEVICE);
522 			if (dma_mapping_error(trans->dev, rxb->page_dma)) {
523 				rxb->page = NULL;
524 				__free_pages(page, trans_pcie->rx_page_order);
525 				continue;
526 			}
527 
528 			/* move the allocated entry to the out list */
529 			list_move(&rxb->list, &local_allocated);
530 			i++;
531 		}
532 
533 		pending--;
534 		if (!pending) {
535 			pending = atomic_xchg(&rba->req_pending, 0);
536 			IWL_DEBUG_RX(trans,
537 				     "Pending allocation requests = %d\n",
538 				     pending);
539 		}
540 
541 		spin_lock(&rba->lock);
542 		/* add the allocated rbds to the allocator allocated list */
543 		list_splice_tail(&local_allocated, &rba->rbd_allocated);
544 		/* get more empty RBDs for current pending requests */
545 		list_splice_tail_init(&rba->rbd_empty, &local_empty);
546 		spin_unlock(&rba->lock);
547 
548 		atomic_inc(&rba->req_ready);
549 	}
550 
551 	spin_lock(&rba->lock);
552 	/* return unused rbds to the allocator empty list */
553 	list_splice_tail(&local_empty, &rba->rbd_empty);
554 	spin_unlock(&rba->lock);
555 }
556 
557 /*
558  * iwl_pcie_rx_allocator_get - returns the pre-allocated pages
559 .*
560 .* Called by queue when the queue posted allocation request and
561  * has freed 8 RBDs in order to restock itself.
562  * This function directly moves the allocated RBs to the queue's ownership
563  * and updates the relevant counters.
564  */
565 static void iwl_pcie_rx_allocator_get(struct iwl_trans *trans,
566 				      struct iwl_rxq *rxq)
567 {
568 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
569 	struct iwl_rb_allocator *rba = &trans_pcie->rba;
570 	int i;
571 
572 	lockdep_assert_held(&rxq->lock);
573 
574 	/*
575 	 * atomic_dec_if_positive returns req_ready - 1 for any scenario.
576 	 * If req_ready is 0 atomic_dec_if_positive will return -1 and this
577 	 * function will return early, as there are no ready requests.
578 	 * atomic_dec_if_positive will perofrm the *actual* decrement only if
579 	 * req_ready > 0, i.e. - there are ready requests and the function
580 	 * hands one request to the caller.
581 	 */
582 	if (atomic_dec_if_positive(&rba->req_ready) < 0)
583 		return;
584 
585 	spin_lock(&rba->lock);
586 	for (i = 0; i < RX_CLAIM_REQ_ALLOC; i++) {
587 		/* Get next free Rx buffer, remove it from free list */
588 		struct iwl_rx_mem_buffer *rxb =
589 			list_first_entry(&rba->rbd_allocated,
590 					 struct iwl_rx_mem_buffer, list);
591 
592 		list_move(&rxb->list, &rxq->rx_free);
593 	}
594 	spin_unlock(&rba->lock);
595 
596 	rxq->used_count -= RX_CLAIM_REQ_ALLOC;
597 	rxq->free_count += RX_CLAIM_REQ_ALLOC;
598 }
599 
600 static void iwl_pcie_rx_allocator_work(struct work_struct *data)
601 {
602 	struct iwl_rb_allocator *rba_p =
603 		container_of(data, struct iwl_rb_allocator, rx_alloc);
604 	struct iwl_trans_pcie *trans_pcie =
605 		container_of(rba_p, struct iwl_trans_pcie, rba);
606 
607 	iwl_pcie_rx_allocator(trans_pcie->trans);
608 }
609 
610 static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
611 {
612 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
613 	struct iwl_rb_allocator *rba = &trans_pcie->rba;
614 	struct device *dev = trans->dev;
615 	int i;
616 	int free_size = trans->cfg->mq_rx_supported ? sizeof(__le64) :
617 						      sizeof(__le32);
618 
619 	if (WARN_ON(trans_pcie->rxq))
620 		return -EINVAL;
621 
622 	trans_pcie->rxq = kcalloc(trans->num_rx_queues, sizeof(struct iwl_rxq),
623 				  GFP_KERNEL);
624 	if (!trans_pcie->rxq)
625 		return -EINVAL;
626 
627 	spin_lock_init(&rba->lock);
628 
629 	for (i = 0; i < trans->num_rx_queues; i++) {
630 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
631 
632 		spin_lock_init(&rxq->lock);
633 		if (trans->cfg->mq_rx_supported)
634 			rxq->queue_size = MQ_RX_TABLE_SIZE;
635 		else
636 			rxq->queue_size = RX_QUEUE_SIZE;
637 
638 		/*
639 		 * Allocate the circular buffer of Read Buffer Descriptors
640 		 * (RBDs)
641 		 */
642 		rxq->bd = dma_zalloc_coherent(dev,
643 					     free_size * rxq->queue_size,
644 					     &rxq->bd_dma, GFP_KERNEL);
645 		if (!rxq->bd)
646 			goto err;
647 
648 		if (trans->cfg->mq_rx_supported) {
649 			rxq->used_bd = dma_zalloc_coherent(dev,
650 							   sizeof(__le32) *
651 							   rxq->queue_size,
652 							   &rxq->used_bd_dma,
653 							   GFP_KERNEL);
654 			if (!rxq->used_bd)
655 				goto err;
656 		}
657 
658 		/*Allocate the driver's pointer to receive buffer status */
659 		rxq->rb_stts = dma_zalloc_coherent(dev, sizeof(*rxq->rb_stts),
660 						   &rxq->rb_stts_dma,
661 						   GFP_KERNEL);
662 		if (!rxq->rb_stts)
663 			goto err;
664 	}
665 	return 0;
666 
667 err:
668 	for (i = 0; i < trans->num_rx_queues; i++) {
669 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
670 
671 		if (rxq->bd)
672 			dma_free_coherent(dev, free_size * rxq->queue_size,
673 					  rxq->bd, rxq->bd_dma);
674 		rxq->bd_dma = 0;
675 		rxq->bd = NULL;
676 
677 		if (rxq->rb_stts)
678 			dma_free_coherent(trans->dev,
679 					  sizeof(struct iwl_rb_status),
680 					  rxq->rb_stts, rxq->rb_stts_dma);
681 
682 		if (rxq->used_bd)
683 			dma_free_coherent(dev, sizeof(__le32) * rxq->queue_size,
684 					  rxq->used_bd, rxq->used_bd_dma);
685 		rxq->used_bd_dma = 0;
686 		rxq->used_bd = NULL;
687 	}
688 	kfree(trans_pcie->rxq);
689 
690 	return -ENOMEM;
691 }
692 
693 static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq)
694 {
695 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
696 	u32 rb_size;
697 	unsigned long flags;
698 	const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */
699 
700 	switch (trans_pcie->rx_buf_size) {
701 	case IWL_AMSDU_4K:
702 		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K;
703 		break;
704 	case IWL_AMSDU_8K:
705 		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K;
706 		break;
707 	case IWL_AMSDU_12K:
708 		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_12K;
709 		break;
710 	default:
711 		WARN_ON(1);
712 		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K;
713 	}
714 
715 	if (!iwl_trans_grab_nic_access(trans, &flags))
716 		return;
717 
718 	/* Stop Rx DMA */
719 	iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
720 	/* reset and flush pointers */
721 	iwl_write32(trans, FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
722 	iwl_write32(trans, FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
723 	iwl_write32(trans, FH_RSCSR_CHNL0_RDPTR, 0);
724 
725 	/* Reset driver's Rx queue write index */
726 	iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
727 
728 	/* Tell device where to find RBD circular buffer in DRAM */
729 	iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG,
730 		    (u32)(rxq->bd_dma >> 8));
731 
732 	/* Tell device where in DRAM to update its Rx status */
733 	iwl_write32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG,
734 		    rxq->rb_stts_dma >> 4);
735 
736 	/* Enable Rx DMA
737 	 * FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
738 	 *      the credit mechanism in 5000 HW RX FIFO
739 	 * Direct rx interrupts to hosts
740 	 * Rx buffer size 4 or 8k or 12k
741 	 * RB timeout 0x10
742 	 * 256 RBDs
743 	 */
744 	iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG,
745 		    FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL |
746 		    FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY |
747 		    FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL |
748 		    rb_size |
749 		    (RX_RB_TIMEOUT << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
750 		    (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS));
751 
752 	iwl_trans_release_nic_access(trans, &flags);
753 
754 	/* Set interrupt coalescing timer to default (2048 usecs) */
755 	iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF);
756 
757 	/* W/A for interrupt coalescing bug in 7260 and 3160 */
758 	if (trans->cfg->host_interrupt_operation_mode)
759 		iwl_set_bit(trans, CSR_INT_COALESCING, IWL_HOST_INT_OPER_MODE);
760 }
761 
762 void iwl_pcie_enable_rx_wake(struct iwl_trans *trans, bool enable)
763 {
764 	/*
765 	 * Turn on the chicken-bits that cause MAC wakeup for RX-related
766 	 * values.
767 	 * This costs some power, but needed for W/A 9000 integrated A-step
768 	 * bug where shadow registers are not in the retention list and their
769 	 * value is lost when NIC powers down
770 	 */
771 	if (trans->cfg->integrated) {
772 		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL,
773 			    CSR_MAC_SHADOW_REG_CTRL_RX_WAKE);
774 		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTL2,
775 			    CSR_MAC_SHADOW_REG_CTL2_RX_WAKE);
776 	}
777 }
778 
779 static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans)
780 {
781 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
782 	u32 rb_size, enabled = 0;
783 	unsigned long flags;
784 	int i;
785 
786 	switch (trans_pcie->rx_buf_size) {
787 	case IWL_AMSDU_4K:
788 		rb_size = RFH_RXF_DMA_RB_SIZE_4K;
789 		break;
790 	case IWL_AMSDU_8K:
791 		rb_size = RFH_RXF_DMA_RB_SIZE_8K;
792 		break;
793 	case IWL_AMSDU_12K:
794 		rb_size = RFH_RXF_DMA_RB_SIZE_12K;
795 		break;
796 	default:
797 		WARN_ON(1);
798 		rb_size = RFH_RXF_DMA_RB_SIZE_4K;
799 	}
800 
801 	if (!iwl_trans_grab_nic_access(trans, &flags))
802 		return;
803 
804 	/* Stop Rx DMA */
805 	iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, 0);
806 	/* disable free amd used rx queue operation */
807 	iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, 0);
808 
809 	for (i = 0; i < trans->num_rx_queues; i++) {
810 		/* Tell device where to find RBD free table in DRAM */
811 		iwl_write_prph64_no_grab(trans,
812 					 RFH_Q_FRBDCB_BA_LSB(i),
813 					 trans_pcie->rxq[i].bd_dma);
814 		/* Tell device where to find RBD used table in DRAM */
815 		iwl_write_prph64_no_grab(trans,
816 					 RFH_Q_URBDCB_BA_LSB(i),
817 					 trans_pcie->rxq[i].used_bd_dma);
818 		/* Tell device where in DRAM to update its Rx status */
819 		iwl_write_prph64_no_grab(trans,
820 					 RFH_Q_URBD_STTS_WPTR_LSB(i),
821 					 trans_pcie->rxq[i].rb_stts_dma);
822 		/* Reset device indice tables */
823 		iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_WIDX(i), 0);
824 		iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_RIDX(i), 0);
825 		iwl_write_prph_no_grab(trans, RFH_Q_URBDCB_WIDX(i), 0);
826 
827 		enabled |= BIT(i) | BIT(i + 16);
828 	}
829 
830 	/*
831 	 * Enable Rx DMA
832 	 * Rx buffer size 4 or 8k or 12k
833 	 * Min RB size 4 or 8
834 	 * Drop frames that exceed RB size
835 	 * 512 RBDs
836 	 */
837 	iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG,
838 			       RFH_DMA_EN_ENABLE_VAL | rb_size |
839 			       RFH_RXF_DMA_MIN_RB_4_8 |
840 			       RFH_RXF_DMA_DROP_TOO_LARGE_MASK |
841 			       RFH_RXF_DMA_RBDCB_SIZE_512);
842 
843 	/*
844 	 * Activate DMA snooping.
845 	 * Set RX DMA chunk size to 64B for IOSF and 128B for PCIe
846 	 * Default queue is 0
847 	 */
848 	iwl_write_prph_no_grab(trans, RFH_GEN_CFG, RFH_GEN_CFG_RFH_DMA_SNOOP |
849 			       (DEFAULT_RXQ_NUM <<
850 				RFH_GEN_CFG_DEFAULT_RXQ_NUM_POS) |
851 			       RFH_GEN_CFG_SERVICE_DMA_SNOOP |
852 			       (trans->cfg->integrated ?
853 				RFH_GEN_CFG_RB_CHUNK_SIZE_64 :
854 				RFH_GEN_CFG_RB_CHUNK_SIZE_128) <<
855 			       RFH_GEN_CFG_RB_CHUNK_SIZE_POS);
856 	/* Enable the relevant rx queues */
857 	iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, enabled);
858 
859 	iwl_trans_release_nic_access(trans, &flags);
860 
861 	/* Set interrupt coalescing timer to default (2048 usecs) */
862 	iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF);
863 
864 	iwl_pcie_enable_rx_wake(trans, true);
865 }
866 
867 static void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq)
868 {
869 	lockdep_assert_held(&rxq->lock);
870 
871 	INIT_LIST_HEAD(&rxq->rx_free);
872 	INIT_LIST_HEAD(&rxq->rx_used);
873 	rxq->free_count = 0;
874 	rxq->used_count = 0;
875 }
876 
877 static int iwl_pcie_dummy_napi_poll(struct napi_struct *napi, int budget)
878 {
879 	WARN_ON(1);
880 	return 0;
881 }
882 
883 int iwl_pcie_rx_init(struct iwl_trans *trans)
884 {
885 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
886 	struct iwl_rxq *def_rxq;
887 	struct iwl_rb_allocator *rba = &trans_pcie->rba;
888 	int i, err, queue_size, allocator_pool_size, num_alloc;
889 
890 	if (!trans_pcie->rxq) {
891 		err = iwl_pcie_rx_alloc(trans);
892 		if (err)
893 			return err;
894 	}
895 	def_rxq = trans_pcie->rxq;
896 	if (!rba->alloc_wq)
897 		rba->alloc_wq = alloc_workqueue("rb_allocator",
898 						WQ_HIGHPRI | WQ_UNBOUND, 1);
899 	INIT_WORK(&rba->rx_alloc, iwl_pcie_rx_allocator_work);
900 
901 	spin_lock(&rba->lock);
902 	atomic_set(&rba->req_pending, 0);
903 	atomic_set(&rba->req_ready, 0);
904 	INIT_LIST_HEAD(&rba->rbd_allocated);
905 	INIT_LIST_HEAD(&rba->rbd_empty);
906 	spin_unlock(&rba->lock);
907 
908 	/* free all first - we might be reconfigured for a different size */
909 	iwl_pcie_free_rbs_pool(trans);
910 
911 	for (i = 0; i < RX_QUEUE_SIZE; i++)
912 		def_rxq->queue[i] = NULL;
913 
914 	for (i = 0; i < trans->num_rx_queues; i++) {
915 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
916 
917 		rxq->id = i;
918 
919 		spin_lock(&rxq->lock);
920 		/*
921 		 * Set read write pointer to reflect that we have processed
922 		 * and used all buffers, but have not restocked the Rx queue
923 		 * with fresh buffers
924 		 */
925 		rxq->read = 0;
926 		rxq->write = 0;
927 		rxq->write_actual = 0;
928 		memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts));
929 
930 		iwl_pcie_rx_init_rxb_lists(rxq);
931 
932 		if (!rxq->napi.poll)
933 			netif_napi_add(&trans_pcie->napi_dev, &rxq->napi,
934 				       iwl_pcie_dummy_napi_poll, 64);
935 
936 		spin_unlock(&rxq->lock);
937 	}
938 
939 	/* move the pool to the default queue and allocator ownerships */
940 	queue_size = trans->cfg->mq_rx_supported ?
941 		     MQ_RX_NUM_RBDS : RX_QUEUE_SIZE;
942 	allocator_pool_size = trans->num_rx_queues *
943 		(RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC);
944 	num_alloc = queue_size + allocator_pool_size;
945 	BUILD_BUG_ON(ARRAY_SIZE(trans_pcie->global_table) !=
946 		     ARRAY_SIZE(trans_pcie->rx_pool));
947 	for (i = 0; i < num_alloc; i++) {
948 		struct iwl_rx_mem_buffer *rxb = &trans_pcie->rx_pool[i];
949 
950 		if (i < allocator_pool_size)
951 			list_add(&rxb->list, &rba->rbd_empty);
952 		else
953 			list_add(&rxb->list, &def_rxq->rx_used);
954 		trans_pcie->global_table[i] = rxb;
955 		rxb->vid = (u16)(i + 1);
956 		rxb->invalid = true;
957 	}
958 
959 	iwl_pcie_rxq_alloc_rbs(trans, GFP_KERNEL, def_rxq);
960 
961 	if (trans->cfg->mq_rx_supported)
962 		iwl_pcie_rx_mq_hw_init(trans);
963 	else
964 		iwl_pcie_rx_hw_init(trans, def_rxq);
965 
966 	iwl_pcie_rxq_restock(trans, def_rxq);
967 
968 	spin_lock(&def_rxq->lock);
969 	iwl_pcie_rxq_inc_wr_ptr(trans, def_rxq);
970 	spin_unlock(&def_rxq->lock);
971 
972 	return 0;
973 }
974 
975 void iwl_pcie_rx_free(struct iwl_trans *trans)
976 {
977 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
978 	struct iwl_rb_allocator *rba = &trans_pcie->rba;
979 	int free_size = trans->cfg->mq_rx_supported ? sizeof(__le64) :
980 					      sizeof(__le32);
981 	int i;
982 
983 	/*
984 	 * if rxq is NULL, it means that nothing has been allocated,
985 	 * exit now
986 	 */
987 	if (!trans_pcie->rxq) {
988 		IWL_DEBUG_INFO(trans, "Free NULL rx context\n");
989 		return;
990 	}
991 
992 	cancel_work_sync(&rba->rx_alloc);
993 	if (rba->alloc_wq) {
994 		destroy_workqueue(rba->alloc_wq);
995 		rba->alloc_wq = NULL;
996 	}
997 
998 	iwl_pcie_free_rbs_pool(trans);
999 
1000 	for (i = 0; i < trans->num_rx_queues; i++) {
1001 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
1002 
1003 		if (rxq->bd)
1004 			dma_free_coherent(trans->dev,
1005 					  free_size * rxq->queue_size,
1006 					  rxq->bd, rxq->bd_dma);
1007 		rxq->bd_dma = 0;
1008 		rxq->bd = NULL;
1009 
1010 		if (rxq->rb_stts)
1011 			dma_free_coherent(trans->dev,
1012 					  sizeof(struct iwl_rb_status),
1013 					  rxq->rb_stts, rxq->rb_stts_dma);
1014 		else
1015 			IWL_DEBUG_INFO(trans,
1016 				       "Free rxq->rb_stts which is NULL\n");
1017 
1018 		if (rxq->used_bd)
1019 			dma_free_coherent(trans->dev,
1020 					  sizeof(__le32) * rxq->queue_size,
1021 					  rxq->used_bd, rxq->used_bd_dma);
1022 		rxq->used_bd_dma = 0;
1023 		rxq->used_bd = NULL;
1024 
1025 		if (rxq->napi.poll)
1026 			netif_napi_del(&rxq->napi);
1027 	}
1028 	kfree(trans_pcie->rxq);
1029 }
1030 
1031 /*
1032  * iwl_pcie_rx_reuse_rbd - Recycle used RBDs
1033  *
1034  * Called when a RBD can be reused. The RBD is transferred to the allocator.
1035  * When there are 2 empty RBDs - a request for allocation is posted
1036  */
1037 static void iwl_pcie_rx_reuse_rbd(struct iwl_trans *trans,
1038 				  struct iwl_rx_mem_buffer *rxb,
1039 				  struct iwl_rxq *rxq, bool emergency)
1040 {
1041 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1042 	struct iwl_rb_allocator *rba = &trans_pcie->rba;
1043 
1044 	/* Move the RBD to the used list, will be moved to allocator in batches
1045 	 * before claiming or posting a request*/
1046 	list_add_tail(&rxb->list, &rxq->rx_used);
1047 
1048 	if (unlikely(emergency))
1049 		return;
1050 
1051 	/* Count the allocator owned RBDs */
1052 	rxq->used_count++;
1053 
1054 	/* If we have RX_POST_REQ_ALLOC new released rx buffers -
1055 	 * issue a request for allocator. Modulo RX_CLAIM_REQ_ALLOC is
1056 	 * used for the case we failed to claim RX_CLAIM_REQ_ALLOC,
1057 	 * after but we still need to post another request.
1058 	 */
1059 	if ((rxq->used_count % RX_CLAIM_REQ_ALLOC) == RX_POST_REQ_ALLOC) {
1060 		/* Move the 2 RBDs to the allocator ownership.
1061 		 Allocator has another 6 from pool for the request completion*/
1062 		spin_lock(&rba->lock);
1063 		list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty);
1064 		spin_unlock(&rba->lock);
1065 
1066 		atomic_inc(&rba->req_pending);
1067 		queue_work(rba->alloc_wq, &rba->rx_alloc);
1068 	}
1069 }
1070 
1071 static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
1072 				struct iwl_rxq *rxq,
1073 				struct iwl_rx_mem_buffer *rxb,
1074 				bool emergency)
1075 {
1076 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1077 	struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
1078 	bool page_stolen = false;
1079 	int max_len = PAGE_SIZE << trans_pcie->rx_page_order;
1080 	u32 offset = 0;
1081 
1082 	if (WARN_ON(!rxb))
1083 		return;
1084 
1085 	dma_unmap_page(trans->dev, rxb->page_dma, max_len, DMA_FROM_DEVICE);
1086 
1087 	while (offset + sizeof(u32) + sizeof(struct iwl_cmd_header) < max_len) {
1088 		struct iwl_rx_packet *pkt;
1089 		u16 sequence;
1090 		bool reclaim;
1091 		int index, cmd_index, len;
1092 		struct iwl_rx_cmd_buffer rxcb = {
1093 			._offset = offset,
1094 			._rx_page_order = trans_pcie->rx_page_order,
1095 			._page = rxb->page,
1096 			._page_stolen = false,
1097 			.truesize = max_len,
1098 		};
1099 
1100 		pkt = rxb_addr(&rxcb);
1101 
1102 		if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID))
1103 			break;
1104 
1105 		WARN_ON((le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >>
1106 			FH_RSCSR_RXQ_POS != rxq->id);
1107 
1108 		IWL_DEBUG_RX(trans,
1109 			     "cmd at offset %d: %s (%.2x.%2x, seq 0x%x)\n",
1110 			     rxcb._offset,
1111 			     iwl_get_cmd_string(trans,
1112 						iwl_cmd_id(pkt->hdr.cmd,
1113 							   pkt->hdr.group_id,
1114 							   0)),
1115 			     pkt->hdr.group_id, pkt->hdr.cmd,
1116 			     le16_to_cpu(pkt->hdr.sequence));
1117 
1118 		len = iwl_rx_packet_len(pkt);
1119 		len += sizeof(u32); /* account for status word */
1120 		trace_iwlwifi_dev_rx(trans->dev, trans, pkt, len);
1121 		trace_iwlwifi_dev_rx_data(trans->dev, trans, pkt, len);
1122 
1123 		/* Reclaim a command buffer only if this packet is a response
1124 		 *   to a (driver-originated) command.
1125 		 * If the packet (e.g. Rx frame) originated from uCode,
1126 		 *   there is no command buffer to reclaim.
1127 		 * Ucode should set SEQ_RX_FRAME bit if ucode-originated,
1128 		 *   but apparently a few don't get set; catch them here. */
1129 		reclaim = !(pkt->hdr.sequence & SEQ_RX_FRAME);
1130 		if (reclaim) {
1131 			int i;
1132 
1133 			for (i = 0; i < trans_pcie->n_no_reclaim_cmds; i++) {
1134 				if (trans_pcie->no_reclaim_cmds[i] ==
1135 							pkt->hdr.cmd) {
1136 					reclaim = false;
1137 					break;
1138 				}
1139 			}
1140 		}
1141 
1142 		sequence = le16_to_cpu(pkt->hdr.sequence);
1143 		index = SEQ_TO_INDEX(sequence);
1144 		cmd_index = get_cmd_index(txq, index);
1145 
1146 		if (rxq->id == 0)
1147 			iwl_op_mode_rx(trans->op_mode, &rxq->napi,
1148 				       &rxcb);
1149 		else
1150 			iwl_op_mode_rx_rss(trans->op_mode, &rxq->napi,
1151 					   &rxcb, rxq->id);
1152 
1153 		if (reclaim) {
1154 			kzfree(txq->entries[cmd_index].free_buf);
1155 			txq->entries[cmd_index].free_buf = NULL;
1156 		}
1157 
1158 		/*
1159 		 * After here, we should always check rxcb._page_stolen,
1160 		 * if it is true then one of the handlers took the page.
1161 		 */
1162 
1163 		if (reclaim) {
1164 			/* Invoke any callbacks, transfer the buffer to caller,
1165 			 * and fire off the (possibly) blocking
1166 			 * iwl_trans_send_cmd()
1167 			 * as we reclaim the driver command queue */
1168 			if (!rxcb._page_stolen)
1169 				iwl_pcie_hcmd_complete(trans, &rxcb);
1170 			else
1171 				IWL_WARN(trans, "Claim null rxb?\n");
1172 		}
1173 
1174 		page_stolen |= rxcb._page_stolen;
1175 		offset += ALIGN(len, FH_RSCSR_FRAME_ALIGN);
1176 	}
1177 
1178 	/* page was stolen from us -- free our reference */
1179 	if (page_stolen) {
1180 		__free_pages(rxb->page, trans_pcie->rx_page_order);
1181 		rxb->page = NULL;
1182 	}
1183 
1184 	/* Reuse the page if possible. For notification packets and
1185 	 * SKBs that fail to Rx correctly, add them back into the
1186 	 * rx_free list for reuse later. */
1187 	if (rxb->page != NULL) {
1188 		rxb->page_dma =
1189 			dma_map_page(trans->dev, rxb->page, 0,
1190 				     PAGE_SIZE << trans_pcie->rx_page_order,
1191 				     DMA_FROM_DEVICE);
1192 		if (dma_mapping_error(trans->dev, rxb->page_dma)) {
1193 			/*
1194 			 * free the page(s) as well to not break
1195 			 * the invariant that the items on the used
1196 			 * list have no page(s)
1197 			 */
1198 			__free_pages(rxb->page, trans_pcie->rx_page_order);
1199 			rxb->page = NULL;
1200 			iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency);
1201 		} else {
1202 			list_add_tail(&rxb->list, &rxq->rx_free);
1203 			rxq->free_count++;
1204 		}
1205 	} else
1206 		iwl_pcie_rx_reuse_rbd(trans, rxb, rxq, emergency);
1207 }
1208 
1209 /*
1210  * iwl_pcie_rx_handle - Main entry function for receiving responses from fw
1211  */
1212 static void iwl_pcie_rx_handle(struct iwl_trans *trans, int queue)
1213 {
1214 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1215 	struct iwl_rxq *rxq = &trans_pcie->rxq[queue];
1216 	u32 r, i, count = 0;
1217 	bool emergency = false;
1218 
1219 restart:
1220 	spin_lock(&rxq->lock);
1221 	/* uCode's read index (stored in shared DRAM) indicates the last Rx
1222 	 * buffer that the driver may process (last buffer filled by ucode). */
1223 	r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF;
1224 	i = rxq->read;
1225 
1226 	/* W/A 9000 device step A0 wrap-around bug */
1227 	r &= (rxq->queue_size - 1);
1228 
1229 	/* Rx interrupt, but nothing sent from uCode */
1230 	if (i == r)
1231 		IWL_DEBUG_RX(trans, "Q %d: HW = SW = %d\n", rxq->id, r);
1232 
1233 	while (i != r) {
1234 		struct iwl_rx_mem_buffer *rxb;
1235 
1236 		if (unlikely(rxq->used_count == rxq->queue_size / 2))
1237 			emergency = true;
1238 
1239 		if (trans->cfg->mq_rx_supported) {
1240 			/*
1241 			 * used_bd is a 32 bit but only 12 are used to retrieve
1242 			 * the vid
1243 			 */
1244 			u16 vid = le32_to_cpu(rxq->used_bd[i]) & 0x0FFF;
1245 
1246 			if (WARN(!vid ||
1247 				 vid > ARRAY_SIZE(trans_pcie->global_table),
1248 				 "Invalid rxb index from HW %u\n", (u32)vid)) {
1249 				iwl_force_nmi(trans);
1250 				goto out;
1251 			}
1252 			rxb = trans_pcie->global_table[vid - 1];
1253 			if (WARN(rxb->invalid,
1254 				 "Invalid rxb from HW %u\n", (u32)vid)) {
1255 				iwl_force_nmi(trans);
1256 				goto out;
1257 			}
1258 			rxb->invalid = true;
1259 		} else {
1260 			rxb = rxq->queue[i];
1261 			rxq->queue[i] = NULL;
1262 		}
1263 
1264 		IWL_DEBUG_RX(trans, "Q %d: HW = %d, SW = %d\n", rxq->id, r, i);
1265 		iwl_pcie_rx_handle_rb(trans, rxq, rxb, emergency);
1266 
1267 		i = (i + 1) & (rxq->queue_size - 1);
1268 
1269 		/*
1270 		 * If we have RX_CLAIM_REQ_ALLOC released rx buffers -
1271 		 * try to claim the pre-allocated buffers from the allocator.
1272 		 * If not ready - will try to reclaim next time.
1273 		 * There is no need to reschedule work - allocator exits only
1274 		 * on success
1275 		 */
1276 		if (rxq->used_count >= RX_CLAIM_REQ_ALLOC)
1277 			iwl_pcie_rx_allocator_get(trans, rxq);
1278 
1279 		if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 && !emergency) {
1280 			struct iwl_rb_allocator *rba = &trans_pcie->rba;
1281 
1282 			/* Add the remaining empty RBDs for allocator use */
1283 			spin_lock(&rba->lock);
1284 			list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty);
1285 			spin_unlock(&rba->lock);
1286 		} else if (emergency) {
1287 			count++;
1288 			if (count == 8) {
1289 				count = 0;
1290 				if (rxq->used_count < rxq->queue_size / 3)
1291 					emergency = false;
1292 
1293 				rxq->read = i;
1294 				spin_unlock(&rxq->lock);
1295 				iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq);
1296 				iwl_pcie_rxq_restock(trans, rxq);
1297 				goto restart;
1298 			}
1299 		}
1300 	}
1301 out:
1302 	/* Backtrack one entry */
1303 	rxq->read = i;
1304 	spin_unlock(&rxq->lock);
1305 
1306 	/*
1307 	 * handle a case where in emergency there are some unallocated RBDs.
1308 	 * those RBDs are in the used list, but are not tracked by the queue's
1309 	 * used_count which counts allocator owned RBDs.
1310 	 * unallocated emergency RBDs must be allocated on exit, otherwise
1311 	 * when called again the function may not be in emergency mode and
1312 	 * they will be handed to the allocator with no tracking in the RBD
1313 	 * allocator counters, which will lead to them never being claimed back
1314 	 * by the queue.
1315 	 * by allocating them here, they are now in the queue free list, and
1316 	 * will be restocked by the next call of iwl_pcie_rxq_restock.
1317 	 */
1318 	if (unlikely(emergency && count))
1319 		iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq);
1320 
1321 	if (rxq->napi.poll)
1322 		napi_gro_flush(&rxq->napi, false);
1323 
1324 	iwl_pcie_rxq_restock(trans, rxq);
1325 }
1326 
1327 static struct iwl_trans_pcie *iwl_pcie_get_trans_pcie(struct msix_entry *entry)
1328 {
1329 	u8 queue = entry->entry;
1330 	struct msix_entry *entries = entry - queue;
1331 
1332 	return container_of(entries, struct iwl_trans_pcie, msix_entries[0]);
1333 }
1334 
1335 static inline void iwl_pcie_clear_irq(struct iwl_trans *trans,
1336 				      struct msix_entry *entry)
1337 {
1338 	/*
1339 	 * Before sending the interrupt the HW disables it to prevent
1340 	 * a nested interrupt. This is done by writing 1 to the corresponding
1341 	 * bit in the mask register. After handling the interrupt, it should be
1342 	 * re-enabled by clearing this bit. This register is defined as
1343 	 * write 1 clear (W1C) register, meaning that it's being clear
1344 	 * by writing 1 to the bit.
1345 	 */
1346 	iwl_write32(trans, CSR_MSIX_AUTOMASK_ST_AD, BIT(entry->entry));
1347 }
1348 
1349 /*
1350  * iwl_pcie_rx_msix_handle - Main entry function for receiving responses from fw
1351  * This interrupt handler should be used with RSS queue only.
1352  */
1353 irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id)
1354 {
1355 	struct msix_entry *entry = dev_id;
1356 	struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry);
1357 	struct iwl_trans *trans = trans_pcie->trans;
1358 
1359 	if (WARN_ON(entry->entry >= trans->num_rx_queues))
1360 		return IRQ_NONE;
1361 
1362 	lock_map_acquire(&trans->sync_cmd_lockdep_map);
1363 
1364 	local_bh_disable();
1365 	iwl_pcie_rx_handle(trans, entry->entry);
1366 	local_bh_enable();
1367 
1368 	iwl_pcie_clear_irq(trans, entry);
1369 
1370 	lock_map_release(&trans->sync_cmd_lockdep_map);
1371 
1372 	return IRQ_HANDLED;
1373 }
1374 
1375 /*
1376  * iwl_pcie_irq_handle_error - called for HW or SW error interrupt from card
1377  */
1378 static void iwl_pcie_irq_handle_error(struct iwl_trans *trans)
1379 {
1380 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1381 	int i;
1382 
1383 	/* W/A for WiFi/WiMAX coex and WiMAX own the RF */
1384 	if (trans->cfg->internal_wimax_coex &&
1385 	    !trans->cfg->apmg_not_supported &&
1386 	    (!(iwl_read_prph(trans, APMG_CLK_CTRL_REG) &
1387 			     APMS_CLK_VAL_MRB_FUNC_MODE) ||
1388 	     (iwl_read_prph(trans, APMG_PS_CTRL_REG) &
1389 			    APMG_PS_CTRL_VAL_RESET_REQ))) {
1390 		clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1391 		iwl_op_mode_wimax_active(trans->op_mode);
1392 		wake_up(&trans_pcie->wait_command_queue);
1393 		return;
1394 	}
1395 
1396 	iwl_pcie_dump_csr(trans);
1397 	iwl_dump_fh(trans, NULL);
1398 
1399 	local_bh_disable();
1400 	/* The STATUS_FW_ERROR bit is set in this function. This must happen
1401 	 * before we wake up the command caller, to ensure a proper cleanup. */
1402 	iwl_trans_fw_error(trans);
1403 	local_bh_enable();
1404 
1405 	for (i = 0; i < trans->cfg->base_params->num_of_queues; i++)
1406 		del_timer(&trans_pcie->txq[i].stuck_timer);
1407 
1408 	clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1409 	wake_up(&trans_pcie->wait_command_queue);
1410 }
1411 
1412 static u32 iwl_pcie_int_cause_non_ict(struct iwl_trans *trans)
1413 {
1414 	u32 inta;
1415 
1416 	lockdep_assert_held(&IWL_TRANS_GET_PCIE_TRANS(trans)->irq_lock);
1417 
1418 	trace_iwlwifi_dev_irq(trans->dev);
1419 
1420 	/* Discover which interrupts are active/pending */
1421 	inta = iwl_read32(trans, CSR_INT);
1422 
1423 	/* the thread will service interrupts and re-enable them */
1424 	return inta;
1425 }
1426 
1427 /* a device (PCI-E) page is 4096 bytes long */
1428 #define ICT_SHIFT	12
1429 #define ICT_SIZE	(1 << ICT_SHIFT)
1430 #define ICT_COUNT	(ICT_SIZE / sizeof(u32))
1431 
1432 /* interrupt handler using ict table, with this interrupt driver will
1433  * stop using INTA register to get device's interrupt, reading this register
1434  * is expensive, device will write interrupts in ICT dram table, increment
1435  * index then will fire interrupt to driver, driver will OR all ICT table
1436  * entries from current index up to table entry with 0 value. the result is
1437  * the interrupt we need to service, driver will set the entries back to 0 and
1438  * set index.
1439  */
1440 static u32 iwl_pcie_int_cause_ict(struct iwl_trans *trans)
1441 {
1442 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1443 	u32 inta;
1444 	u32 val = 0;
1445 	u32 read;
1446 
1447 	trace_iwlwifi_dev_irq(trans->dev);
1448 
1449 	/* Ignore interrupt if there's nothing in NIC to service.
1450 	 * This may be due to IRQ shared with another device,
1451 	 * or due to sporadic interrupts thrown from our NIC. */
1452 	read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]);
1453 	trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index, read);
1454 	if (!read)
1455 		return 0;
1456 
1457 	/*
1458 	 * Collect all entries up to the first 0, starting from ict_index;
1459 	 * note we already read at ict_index.
1460 	 */
1461 	do {
1462 		val |= read;
1463 		IWL_DEBUG_ISR(trans, "ICT index %d value 0x%08X\n",
1464 				trans_pcie->ict_index, read);
1465 		trans_pcie->ict_tbl[trans_pcie->ict_index] = 0;
1466 		trans_pcie->ict_index =
1467 			((trans_pcie->ict_index + 1) & (ICT_COUNT - 1));
1468 
1469 		read = le32_to_cpu(trans_pcie->ict_tbl[trans_pcie->ict_index]);
1470 		trace_iwlwifi_dev_ict_read(trans->dev, trans_pcie->ict_index,
1471 					   read);
1472 	} while (read);
1473 
1474 	/* We should not get this value, just ignore it. */
1475 	if (val == 0xffffffff)
1476 		val = 0;
1477 
1478 	/*
1479 	 * this is a w/a for a h/w bug. the h/w bug may cause the Rx bit
1480 	 * (bit 15 before shifting it to 31) to clear when using interrupt
1481 	 * coalescing. fortunately, bits 18 and 19 stay set when this happens
1482 	 * so we use them to decide on the real state of the Rx bit.
1483 	 * In order words, bit 15 is set if bit 18 or bit 19 are set.
1484 	 */
1485 	if (val & 0xC0000)
1486 		val |= 0x8000;
1487 
1488 	inta = (0xff & val) | ((0xff00 & val) << 16);
1489 	return inta;
1490 }
1491 
1492 irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
1493 {
1494 	struct iwl_trans *trans = dev_id;
1495 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1496 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
1497 	u32 inta = 0;
1498 	u32 handled = 0;
1499 
1500 	lock_map_acquire(&trans->sync_cmd_lockdep_map);
1501 
1502 	spin_lock(&trans_pcie->irq_lock);
1503 
1504 	/* dram interrupt table not set yet,
1505 	 * use legacy interrupt.
1506 	 */
1507 	if (likely(trans_pcie->use_ict))
1508 		inta = iwl_pcie_int_cause_ict(trans);
1509 	else
1510 		inta = iwl_pcie_int_cause_non_ict(trans);
1511 
1512 	if (iwl_have_debug_level(IWL_DL_ISR)) {
1513 		IWL_DEBUG_ISR(trans,
1514 			      "ISR inta 0x%08x, enabled 0x%08x(sw), enabled(hw) 0x%08x, fh 0x%08x\n",
1515 			      inta, trans_pcie->inta_mask,
1516 			      iwl_read32(trans, CSR_INT_MASK),
1517 			      iwl_read32(trans, CSR_FH_INT_STATUS));
1518 		if (inta & (~trans_pcie->inta_mask))
1519 			IWL_DEBUG_ISR(trans,
1520 				      "We got a masked interrupt (0x%08x)\n",
1521 				      inta & (~trans_pcie->inta_mask));
1522 	}
1523 
1524 	inta &= trans_pcie->inta_mask;
1525 
1526 	/*
1527 	 * Ignore interrupt if there's nothing in NIC to service.
1528 	 * This may be due to IRQ shared with another device,
1529 	 * or due to sporadic interrupts thrown from our NIC.
1530 	 */
1531 	if (unlikely(!inta)) {
1532 		IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n");
1533 		/*
1534 		 * Re-enable interrupts here since we don't
1535 		 * have anything to service
1536 		 */
1537 		if (test_bit(STATUS_INT_ENABLED, &trans->status))
1538 			_iwl_enable_interrupts(trans);
1539 		spin_unlock(&trans_pcie->irq_lock);
1540 		lock_map_release(&trans->sync_cmd_lockdep_map);
1541 		return IRQ_NONE;
1542 	}
1543 
1544 	if (unlikely(inta == 0xFFFFFFFF || (inta & 0xFFFFFFF0) == 0xa5a5a5a0)) {
1545 		/*
1546 		 * Hardware disappeared. It might have
1547 		 * already raised an interrupt.
1548 		 */
1549 		IWL_WARN(trans, "HARDWARE GONE?? INTA == 0x%08x\n", inta);
1550 		spin_unlock(&trans_pcie->irq_lock);
1551 		goto out;
1552 	}
1553 
1554 	/* Ack/clear/reset pending uCode interrupts.
1555 	 * Note:  Some bits in CSR_INT are "OR" of bits in CSR_FH_INT_STATUS,
1556 	 */
1557 	/* There is a hardware bug in the interrupt mask function that some
1558 	 * interrupts (i.e. CSR_INT_BIT_SCD) can still be generated even if
1559 	 * they are disabled in the CSR_INT_MASK register. Furthermore the
1560 	 * ICT interrupt handling mechanism has another bug that might cause
1561 	 * these unmasked interrupts fail to be detected. We workaround the
1562 	 * hardware bugs here by ACKing all the possible interrupts so that
1563 	 * interrupt coalescing can still be achieved.
1564 	 */
1565 	iwl_write32(trans, CSR_INT, inta | ~trans_pcie->inta_mask);
1566 
1567 	if (iwl_have_debug_level(IWL_DL_ISR))
1568 		IWL_DEBUG_ISR(trans, "inta 0x%08x, enabled 0x%08x\n",
1569 			      inta, iwl_read32(trans, CSR_INT_MASK));
1570 
1571 	spin_unlock(&trans_pcie->irq_lock);
1572 
1573 	/* Now service all interrupt bits discovered above. */
1574 	if (inta & CSR_INT_BIT_HW_ERR) {
1575 		IWL_ERR(trans, "Hardware error detected.  Restarting.\n");
1576 
1577 		/* Tell the device to stop sending interrupts */
1578 		iwl_disable_interrupts(trans);
1579 
1580 		isr_stats->hw++;
1581 		iwl_pcie_irq_handle_error(trans);
1582 
1583 		handled |= CSR_INT_BIT_HW_ERR;
1584 
1585 		goto out;
1586 	}
1587 
1588 	if (iwl_have_debug_level(IWL_DL_ISR)) {
1589 		/* NIC fires this, but we don't use it, redundant with WAKEUP */
1590 		if (inta & CSR_INT_BIT_SCD) {
1591 			IWL_DEBUG_ISR(trans,
1592 				      "Scheduler finished to transmit the frame/frames.\n");
1593 			isr_stats->sch++;
1594 		}
1595 
1596 		/* Alive notification via Rx interrupt will do the real work */
1597 		if (inta & CSR_INT_BIT_ALIVE) {
1598 			IWL_DEBUG_ISR(trans, "Alive interrupt\n");
1599 			isr_stats->alive++;
1600 		}
1601 	}
1602 
1603 	/* Safely ignore these bits for debug checks below */
1604 	inta &= ~(CSR_INT_BIT_SCD | CSR_INT_BIT_ALIVE);
1605 
1606 	/* HW RF KILL switch toggled */
1607 	if (inta & CSR_INT_BIT_RF_KILL) {
1608 		bool hw_rfkill;
1609 
1610 		mutex_lock(&trans_pcie->mutex);
1611 		hw_rfkill = iwl_is_rfkill_set(trans);
1612 		if (hw_rfkill)
1613 			set_bit(STATUS_RFKILL, &trans->status);
1614 
1615 		IWL_WARN(trans, "RF_KILL bit toggled to %s.\n",
1616 			 hw_rfkill ? "disable radio" : "enable radio");
1617 
1618 		isr_stats->rfkill++;
1619 
1620 		iwl_trans_pcie_rf_kill(trans, hw_rfkill);
1621 		mutex_unlock(&trans_pcie->mutex);
1622 		if (hw_rfkill) {
1623 			if (test_and_clear_bit(STATUS_SYNC_HCMD_ACTIVE,
1624 					       &trans->status))
1625 				IWL_DEBUG_RF_KILL(trans,
1626 						  "Rfkill while SYNC HCMD in flight\n");
1627 			wake_up(&trans_pcie->wait_command_queue);
1628 		} else {
1629 			clear_bit(STATUS_RFKILL, &trans->status);
1630 		}
1631 
1632 		handled |= CSR_INT_BIT_RF_KILL;
1633 	}
1634 
1635 	/* Chip got too hot and stopped itself */
1636 	if (inta & CSR_INT_BIT_CT_KILL) {
1637 		IWL_ERR(trans, "Microcode CT kill error detected.\n");
1638 		isr_stats->ctkill++;
1639 		handled |= CSR_INT_BIT_CT_KILL;
1640 	}
1641 
1642 	/* Error detected by uCode */
1643 	if (inta & CSR_INT_BIT_SW_ERR) {
1644 		IWL_ERR(trans, "Microcode SW error detected. "
1645 			" Restarting 0x%X.\n", inta);
1646 		isr_stats->sw++;
1647 		iwl_pcie_irq_handle_error(trans);
1648 		handled |= CSR_INT_BIT_SW_ERR;
1649 	}
1650 
1651 	/* uCode wakes up after power-down sleep */
1652 	if (inta & CSR_INT_BIT_WAKEUP) {
1653 		IWL_DEBUG_ISR(trans, "Wakeup interrupt\n");
1654 		iwl_pcie_rxq_check_wrptr(trans);
1655 		iwl_pcie_txq_check_wrptrs(trans);
1656 
1657 		isr_stats->wakeup++;
1658 
1659 		handled |= CSR_INT_BIT_WAKEUP;
1660 	}
1661 
1662 	/* All uCode command responses, including Tx command responses,
1663 	 * Rx "responses" (frame-received notification), and other
1664 	 * notifications from uCode come through here*/
1665 	if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX |
1666 		    CSR_INT_BIT_RX_PERIODIC)) {
1667 		IWL_DEBUG_ISR(trans, "Rx interrupt\n");
1668 		if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX)) {
1669 			handled |= (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX);
1670 			iwl_write32(trans, CSR_FH_INT_STATUS,
1671 					CSR_FH_INT_RX_MASK);
1672 		}
1673 		if (inta & CSR_INT_BIT_RX_PERIODIC) {
1674 			handled |= CSR_INT_BIT_RX_PERIODIC;
1675 			iwl_write32(trans,
1676 				CSR_INT, CSR_INT_BIT_RX_PERIODIC);
1677 		}
1678 		/* Sending RX interrupt require many steps to be done in the
1679 		 * the device:
1680 		 * 1- write interrupt to current index in ICT table.
1681 		 * 2- dma RX frame.
1682 		 * 3- update RX shared data to indicate last write index.
1683 		 * 4- send interrupt.
1684 		 * This could lead to RX race, driver could receive RX interrupt
1685 		 * but the shared data changes does not reflect this;
1686 		 * periodic interrupt will detect any dangling Rx activity.
1687 		 */
1688 
1689 		/* Disable periodic interrupt; we use it as just a one-shot. */
1690 		iwl_write8(trans, CSR_INT_PERIODIC_REG,
1691 			    CSR_INT_PERIODIC_DIS);
1692 
1693 		/*
1694 		 * Enable periodic interrupt in 8 msec only if we received
1695 		 * real RX interrupt (instead of just periodic int), to catch
1696 		 * any dangling Rx interrupt.  If it was just the periodic
1697 		 * interrupt, there was no dangling Rx activity, and no need
1698 		 * to extend the periodic interrupt; one-shot is enough.
1699 		 */
1700 		if (inta & (CSR_INT_BIT_FH_RX | CSR_INT_BIT_SW_RX))
1701 			iwl_write8(trans, CSR_INT_PERIODIC_REG,
1702 				   CSR_INT_PERIODIC_ENA);
1703 
1704 		isr_stats->rx++;
1705 
1706 		local_bh_disable();
1707 		iwl_pcie_rx_handle(trans, 0);
1708 		local_bh_enable();
1709 	}
1710 
1711 	/* This "Tx" DMA channel is used only for loading uCode */
1712 	if (inta & CSR_INT_BIT_FH_TX) {
1713 		iwl_write32(trans, CSR_FH_INT_STATUS, CSR_FH_INT_TX_MASK);
1714 		IWL_DEBUG_ISR(trans, "uCode load interrupt\n");
1715 		isr_stats->tx++;
1716 		handled |= CSR_INT_BIT_FH_TX;
1717 		/* Wake up uCode load routine, now that load is complete */
1718 		trans_pcie->ucode_write_complete = true;
1719 		wake_up(&trans_pcie->ucode_write_waitq);
1720 	}
1721 
1722 	if (inta & ~handled) {
1723 		IWL_ERR(trans, "Unhandled INTA bits 0x%08x\n", inta & ~handled);
1724 		isr_stats->unhandled++;
1725 	}
1726 
1727 	if (inta & ~(trans_pcie->inta_mask)) {
1728 		IWL_WARN(trans, "Disabled INTA bits 0x%08x were pending\n",
1729 			 inta & ~trans_pcie->inta_mask);
1730 	}
1731 
1732 	spin_lock(&trans_pcie->irq_lock);
1733 	/* only Re-enable all interrupt if disabled by irq */
1734 	if (test_bit(STATUS_INT_ENABLED, &trans->status))
1735 		_iwl_enable_interrupts(trans);
1736 	/* we are loading the firmware, enable FH_TX interrupt only */
1737 	else if (handled & CSR_INT_BIT_FH_TX)
1738 		iwl_enable_fw_load_int(trans);
1739 	/* Re-enable RF_KILL if it occurred */
1740 	else if (handled & CSR_INT_BIT_RF_KILL)
1741 		iwl_enable_rfkill_int(trans);
1742 	spin_unlock(&trans_pcie->irq_lock);
1743 
1744 out:
1745 	lock_map_release(&trans->sync_cmd_lockdep_map);
1746 	return IRQ_HANDLED;
1747 }
1748 
1749 /******************************************************************************
1750  *
1751  * ICT functions
1752  *
1753  ******************************************************************************/
1754 
1755 /* Free dram table */
1756 void iwl_pcie_free_ict(struct iwl_trans *trans)
1757 {
1758 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1759 
1760 	if (trans_pcie->ict_tbl) {
1761 		dma_free_coherent(trans->dev, ICT_SIZE,
1762 				  trans_pcie->ict_tbl,
1763 				  trans_pcie->ict_tbl_dma);
1764 		trans_pcie->ict_tbl = NULL;
1765 		trans_pcie->ict_tbl_dma = 0;
1766 	}
1767 }
1768 
1769 /*
1770  * allocate dram shared table, it is an aligned memory
1771  * block of ICT_SIZE.
1772  * also reset all data related to ICT table interrupt.
1773  */
1774 int iwl_pcie_alloc_ict(struct iwl_trans *trans)
1775 {
1776 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1777 
1778 	trans_pcie->ict_tbl =
1779 		dma_zalloc_coherent(trans->dev, ICT_SIZE,
1780 				   &trans_pcie->ict_tbl_dma,
1781 				   GFP_KERNEL);
1782 	if (!trans_pcie->ict_tbl)
1783 		return -ENOMEM;
1784 
1785 	/* just an API sanity check ... it is guaranteed to be aligned */
1786 	if (WARN_ON(trans_pcie->ict_tbl_dma & (ICT_SIZE - 1))) {
1787 		iwl_pcie_free_ict(trans);
1788 		return -EINVAL;
1789 	}
1790 
1791 	return 0;
1792 }
1793 
1794 /* Device is going up inform it about using ICT interrupt table,
1795  * also we need to tell the driver to start using ICT interrupt.
1796  */
1797 void iwl_pcie_reset_ict(struct iwl_trans *trans)
1798 {
1799 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1800 	u32 val;
1801 
1802 	if (!trans_pcie->ict_tbl)
1803 		return;
1804 
1805 	spin_lock(&trans_pcie->irq_lock);
1806 	_iwl_disable_interrupts(trans);
1807 
1808 	memset(trans_pcie->ict_tbl, 0, ICT_SIZE);
1809 
1810 	val = trans_pcie->ict_tbl_dma >> ICT_SHIFT;
1811 
1812 	val |= CSR_DRAM_INT_TBL_ENABLE |
1813 	       CSR_DRAM_INIT_TBL_WRAP_CHECK |
1814 	       CSR_DRAM_INIT_TBL_WRITE_POINTER;
1815 
1816 	IWL_DEBUG_ISR(trans, "CSR_DRAM_INT_TBL_REG =0x%x\n", val);
1817 
1818 	iwl_write32(trans, CSR_DRAM_INT_TBL_REG, val);
1819 	trans_pcie->use_ict = true;
1820 	trans_pcie->ict_index = 0;
1821 	iwl_write32(trans, CSR_INT, trans_pcie->inta_mask);
1822 	_iwl_enable_interrupts(trans);
1823 	spin_unlock(&trans_pcie->irq_lock);
1824 }
1825 
1826 /* Device is going down disable ict interrupt usage */
1827 void iwl_pcie_disable_ict(struct iwl_trans *trans)
1828 {
1829 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1830 
1831 	spin_lock(&trans_pcie->irq_lock);
1832 	trans_pcie->use_ict = false;
1833 	spin_unlock(&trans_pcie->irq_lock);
1834 }
1835 
1836 irqreturn_t iwl_pcie_isr(int irq, void *data)
1837 {
1838 	struct iwl_trans *trans = data;
1839 
1840 	if (!trans)
1841 		return IRQ_NONE;
1842 
1843 	/* Disable (but don't clear!) interrupts here to avoid
1844 	 * back-to-back ISRs and sporadic interrupts from our NIC.
1845 	 * If we have something to service, the tasklet will re-enable ints.
1846 	 * If we *don't* have something, we'll re-enable before leaving here.
1847 	 */
1848 	iwl_write32(trans, CSR_INT_MASK, 0x00000000);
1849 
1850 	return IRQ_WAKE_THREAD;
1851 }
1852 
1853 irqreturn_t iwl_pcie_msix_isr(int irq, void *data)
1854 {
1855 	return IRQ_WAKE_THREAD;
1856 }
1857 
1858 irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
1859 {
1860 	struct msix_entry *entry = dev_id;
1861 	struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry);
1862 	struct iwl_trans *trans = trans_pcie->trans;
1863 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
1864 	u32 inta_fh, inta_hw;
1865 
1866 	lock_map_acquire(&trans->sync_cmd_lockdep_map);
1867 
1868 	spin_lock(&trans_pcie->irq_lock);
1869 	inta_fh = iwl_read32(trans, CSR_MSIX_FH_INT_CAUSES_AD);
1870 	inta_hw = iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD);
1871 	/*
1872 	 * Clear causes registers to avoid being handling the same cause.
1873 	 */
1874 	iwl_write32(trans, CSR_MSIX_FH_INT_CAUSES_AD, inta_fh);
1875 	iwl_write32(trans, CSR_MSIX_HW_INT_CAUSES_AD, inta_hw);
1876 	spin_unlock(&trans_pcie->irq_lock);
1877 
1878 	if (unlikely(!(inta_fh | inta_hw))) {
1879 		IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n");
1880 		lock_map_release(&trans->sync_cmd_lockdep_map);
1881 		return IRQ_NONE;
1882 	}
1883 
1884 	if (iwl_have_debug_level(IWL_DL_ISR))
1885 		IWL_DEBUG_ISR(trans, "ISR inta_fh 0x%08x, enabled 0x%08x\n",
1886 			      inta_fh,
1887 			      iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD));
1888 
1889 	if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) &&
1890 	    inta_fh & MSIX_FH_INT_CAUSES_Q0) {
1891 		local_bh_disable();
1892 		iwl_pcie_rx_handle(trans, 0);
1893 		local_bh_enable();
1894 	}
1895 
1896 	if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) &&
1897 	    inta_fh & MSIX_FH_INT_CAUSES_Q1) {
1898 		local_bh_disable();
1899 		iwl_pcie_rx_handle(trans, 1);
1900 		local_bh_enable();
1901 	}
1902 
1903 	/* This "Tx" DMA channel is used only for loading uCode */
1904 	if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) {
1905 		IWL_DEBUG_ISR(trans, "uCode load interrupt\n");
1906 		isr_stats->tx++;
1907 		/*
1908 		 * Wake up uCode load routine,
1909 		 * now that load is complete
1910 		 */
1911 		trans_pcie->ucode_write_complete = true;
1912 		wake_up(&trans_pcie->ucode_write_waitq);
1913 	}
1914 
1915 	/* Error detected by uCode */
1916 	if ((inta_fh & MSIX_FH_INT_CAUSES_FH_ERR) ||
1917 	    (inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR)) {
1918 		IWL_ERR(trans,
1919 			"Microcode SW error detected. Restarting 0x%X.\n",
1920 			inta_fh);
1921 		isr_stats->sw++;
1922 		iwl_pcie_irq_handle_error(trans);
1923 	}
1924 
1925 	/* After checking FH register check HW register */
1926 	if (iwl_have_debug_level(IWL_DL_ISR))
1927 		IWL_DEBUG_ISR(trans,
1928 			      "ISR inta_hw 0x%08x, enabled 0x%08x\n",
1929 			      inta_hw,
1930 			      iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD));
1931 
1932 	/* Alive notification via Rx interrupt will do the real work */
1933 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_ALIVE) {
1934 		IWL_DEBUG_ISR(trans, "Alive interrupt\n");
1935 		isr_stats->alive++;
1936 	}
1937 
1938 	/* uCode wakes up after power-down sleep */
1939 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP) {
1940 		IWL_DEBUG_ISR(trans, "Wakeup interrupt\n");
1941 		iwl_pcie_rxq_check_wrptr(trans);
1942 		iwl_pcie_txq_check_wrptrs(trans);
1943 
1944 		isr_stats->wakeup++;
1945 	}
1946 
1947 	/* Chip got too hot and stopped itself */
1948 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_CT_KILL) {
1949 		IWL_ERR(trans, "Microcode CT kill error detected.\n");
1950 		isr_stats->ctkill++;
1951 	}
1952 
1953 	/* HW RF KILL switch toggled */
1954 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_RF_KILL) {
1955 		bool hw_rfkill;
1956 
1957 		mutex_lock(&trans_pcie->mutex);
1958 		hw_rfkill = iwl_is_rfkill_set(trans);
1959 		if (hw_rfkill)
1960 			set_bit(STATUS_RFKILL, &trans->status);
1961 
1962 		IWL_WARN(trans, "RF_KILL bit toggled to %s.\n",
1963 			 hw_rfkill ? "disable radio" : "enable radio");
1964 
1965 		isr_stats->rfkill++;
1966 
1967 		iwl_trans_pcie_rf_kill(trans, hw_rfkill);
1968 		mutex_unlock(&trans_pcie->mutex);
1969 		if (hw_rfkill) {
1970 			if (test_and_clear_bit(STATUS_SYNC_HCMD_ACTIVE,
1971 					       &trans->status))
1972 				IWL_DEBUG_RF_KILL(trans,
1973 						  "Rfkill while SYNC HCMD in flight\n");
1974 			wake_up(&trans_pcie->wait_command_queue);
1975 		} else {
1976 			clear_bit(STATUS_RFKILL, &trans->status);
1977 		}
1978 	}
1979 
1980 	if (inta_hw & MSIX_HW_INT_CAUSES_REG_HW_ERR) {
1981 		IWL_ERR(trans,
1982 			"Hardware error detected. Restarting.\n");
1983 
1984 		isr_stats->hw++;
1985 		iwl_pcie_irq_handle_error(trans);
1986 	}
1987 
1988 	iwl_pcie_clear_irq(trans, entry);
1989 
1990 	lock_map_release(&trans->sync_cmd_lockdep_map);
1991 
1992 	return IRQ_HANDLED;
1993 }
1994