xref: /freebsd/sys/contrib/dev/athk/ath11k/pcic.c (revision 22cf89c938886d14f5796fc49f9f020c23ea8eaf)
1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
4  * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved.
5  */
6 
7 #include "core.h"
8 #include "pcic.h"
9 #include "debug.h"
10 
11 static const char *irq_name[ATH11K_IRQ_NUM_MAX] = {
12 	"bhi",
13 	"mhi-er0",
14 	"mhi-er1",
15 	"ce0",
16 	"ce1",
17 	"ce2",
18 	"ce3",
19 	"ce4",
20 	"ce5",
21 	"ce6",
22 	"ce7",
23 	"ce8",
24 	"ce9",
25 	"ce10",
26 	"ce11",
27 	"host2wbm-desc-feed",
28 	"host2reo-re-injection",
29 	"host2reo-command",
30 	"host2rxdma-monitor-ring3",
31 	"host2rxdma-monitor-ring2",
32 	"host2rxdma-monitor-ring1",
33 	"reo2ost-exception",
34 	"wbm2host-rx-release",
35 	"reo2host-status",
36 	"reo2host-destination-ring4",
37 	"reo2host-destination-ring3",
38 	"reo2host-destination-ring2",
39 	"reo2host-destination-ring1",
40 	"rxdma2host-monitor-destination-mac3",
41 	"rxdma2host-monitor-destination-mac2",
42 	"rxdma2host-monitor-destination-mac1",
43 	"ppdu-end-interrupts-mac3",
44 	"ppdu-end-interrupts-mac2",
45 	"ppdu-end-interrupts-mac1",
46 	"rxdma2host-monitor-status-ring-mac3",
47 	"rxdma2host-monitor-status-ring-mac2",
48 	"rxdma2host-monitor-status-ring-mac1",
49 	"host2rxdma-host-buf-ring-mac3",
50 	"host2rxdma-host-buf-ring-mac2",
51 	"host2rxdma-host-buf-ring-mac1",
52 	"rxdma2host-destination-ring-mac3",
53 	"rxdma2host-destination-ring-mac2",
54 	"rxdma2host-destination-ring-mac1",
55 	"host2tcl-input-ring4",
56 	"host2tcl-input-ring3",
57 	"host2tcl-input-ring2",
58 	"host2tcl-input-ring1",
59 	"wbm2host-tx-completions-ring3",
60 	"wbm2host-tx-completions-ring2",
61 	"wbm2host-tx-completions-ring1",
62 	"tcl2host-status-ring",
63 };
64 
65 static const struct ath11k_msi_config ath11k_msi_config[] = {
66 	{
67 		.total_vectors = 32,
68 		.total_users = 4,
69 		.users = (struct ath11k_msi_user[]) {
70 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
71 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
72 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
73 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
74 		},
75 		.hw_rev = ATH11K_HW_QCA6390_HW20,
76 	},
77 	{
78 		.total_vectors = 16,
79 		.total_users = 3,
80 		.users = (struct ath11k_msi_user[]) {
81 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
82 			{ .name = "CE", .num_vectors = 5, .base_vector = 3 },
83 			{ .name = "DP", .num_vectors = 8, .base_vector = 8 },
84 		},
85 		.hw_rev = ATH11K_HW_QCN9074_HW10,
86 	},
87 	{
88 		.total_vectors = 32,
89 		.total_users = 4,
90 		.users = (struct ath11k_msi_user[]) {
91 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
92 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
93 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
94 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
95 		},
96 		.hw_rev = ATH11K_HW_WCN6855_HW20,
97 	},
98 	{
99 		.total_vectors = 32,
100 		.total_users = 4,
101 		.users = (struct ath11k_msi_user[]) {
102 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
103 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
104 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
105 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
106 		},
107 		.hw_rev = ATH11K_HW_WCN6855_HW21,
108 	},
109 	{
110 		.total_vectors = 28,
111 		.total_users = 2,
112 		.users = (struct ath11k_msi_user[]) {
113 			{ .name = "CE", .num_vectors = 10, .base_vector = 0 },
114 			{ .name = "DP", .num_vectors = 18, .base_vector = 10 },
115 		},
116 		.hw_rev = ATH11K_HW_WCN6750_HW10,
117 	},
118 };
119 
120 int ath11k_pcic_init_msi_config(struct ath11k_base *ab)
121 {
122 	const struct ath11k_msi_config *msi_config;
123 	int i;
124 
125 	for (i = 0; i < ARRAY_SIZE(ath11k_msi_config); i++) {
126 		msi_config = &ath11k_msi_config[i];
127 
128 		if (msi_config->hw_rev == ab->hw_rev)
129 			break;
130 	}
131 
132 	if (i == ARRAY_SIZE(ath11k_msi_config)) {
133 		ath11k_err(ab, "failed to fetch msi config, unsupported hw version: 0x%x\n",
134 			   ab->hw_rev);
135 		return -EINVAL;
136 	}
137 
138 	ab->pci.msi.config = msi_config;
139 	return 0;
140 }
141 EXPORT_SYMBOL(ath11k_pcic_init_msi_config);
142 
143 static void __ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
144 {
145 	if (offset < ATH11K_PCI_WINDOW_START)
146 #if defined(__linux__)
147 		iowrite32(value, ab->mem  + offset);
148 #elif defined(__FreeBSD__)
149 		iowrite32(value, (char *)ab->mem  + offset);
150 #endif
151 	else
152 		ab->pci.ops->window_write32(ab, offset, value);
153 }
154 
155 void ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
156 {
157 	int ret = 0;
158 	bool wakeup_required;
159 
160 	/* for offset beyond BAR + 4K - 32, may
161 	 * need to wakeup the device to access.
162 	 */
163 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
164 			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
165 	if (wakeup_required && ab->pci.ops->wakeup)
166 		ret = ab->pci.ops->wakeup(ab);
167 
168 	__ath11k_pcic_write32(ab, offset, value);
169 
170 	if (wakeup_required && !ret && ab->pci.ops->release)
171 		ab->pci.ops->release(ab);
172 }
173 EXPORT_SYMBOL(ath11k_pcic_write32);
174 
175 static u32 __ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
176 {
177 	u32 val;
178 
179 	if (offset < ATH11K_PCI_WINDOW_START)
180 #if defined(__linux__)
181 		val = ioread32(ab->mem + offset);
182 #elif defined(__FreeBSD__)
183 		val = ioread32((char *)ab->mem + offset);
184 #endif
185 	else
186 		val = ab->pci.ops->window_read32(ab, offset);
187 
188 	return val;
189 }
190 
191 u32 ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
192 {
193 	int ret = 0;
194 	u32 val;
195 	bool wakeup_required;
196 
197 	/* for offset beyond BAR + 4K - 32, may
198 	 * need to wakeup the device to access.
199 	 */
200 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
201 			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
202 	if (wakeup_required && ab->pci.ops->wakeup)
203 		ret = ab->pci.ops->wakeup(ab);
204 
205 	val = __ath11k_pcic_read32(ab, offset);
206 
207 	if (wakeup_required && !ret && ab->pci.ops->release)
208 		ab->pci.ops->release(ab);
209 
210 	return val;
211 }
212 EXPORT_SYMBOL(ath11k_pcic_read32);
213 
214 int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end)
215 {
216 	int ret = 0;
217 	bool wakeup_required;
218 	u32 *data = buf;
219 	u32 i;
220 
221 	/* for offset beyond BAR + 4K - 32, may
222 	 * need to wakeup the device to access.
223 	 */
224 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
225 			  end >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
226 	if (wakeup_required && ab->pci.ops->wakeup) {
227 		ret = ab->pci.ops->wakeup(ab);
228 		if (ret) {
229 			ath11k_warn(ab,
230 				    "wakeup failed, data may be invalid: %d",
231 				    ret);
232 			/* Even though wakeup() failed, continue processing rather
233 			 * than returning because some parts of the data may still
234 			 * be valid and useful in some cases, e.g. could give us
235 			 * some clues on firmware crash.
236 			 * Mislead due to invalid data could be avoided because we
237 			 * are aware of the wakeup failure.
238 			 */
239 		}
240 	}
241 
242 	for (i = start; i < end + 1; i += 4)
243 		*data++ = __ath11k_pcic_read32(ab, i);
244 
245 	if (wakeup_required && ab->pci.ops->release)
246 		ab->pci.ops->release(ab);
247 
248 	return 0;
249 }
250 EXPORT_SYMBOL(ath11k_pcic_read);
251 
252 void ath11k_pcic_get_msi_address(struct ath11k_base *ab, u32 *msi_addr_lo,
253 				 u32 *msi_addr_hi)
254 {
255 	*msi_addr_lo = ab->pci.msi.addr_lo;
256 	*msi_addr_hi = ab->pci.msi.addr_hi;
257 }
258 EXPORT_SYMBOL(ath11k_pcic_get_msi_address);
259 
260 int ath11k_pcic_get_user_msi_assignment(struct ath11k_base *ab, char *user_name,
261 					int *num_vectors, u32 *user_base_data,
262 					u32 *base_vector)
263 {
264 	const struct ath11k_msi_config *msi_config = ab->pci.msi.config;
265 	int idx;
266 
267 	for (idx = 0; idx < msi_config->total_users; idx++) {
268 		if (strcmp(user_name, msi_config->users[idx].name) == 0) {
269 			*num_vectors = msi_config->users[idx].num_vectors;
270 			*base_vector =  msi_config->users[idx].base_vector;
271 			*user_base_data = *base_vector + ab->pci.msi.ep_base_data;
272 
273 			ath11k_dbg(ab, ATH11K_DBG_PCI,
274 				   "msi assignment %s num_vectors %d user_base_data %u base_vector %u\n",
275 				   user_name, *num_vectors, *user_base_data,
276 				   *base_vector);
277 
278 			return 0;
279 		}
280 	}
281 
282 	ath11k_err(ab, "Failed to find MSI assignment for %s!\n", user_name);
283 
284 	return -EINVAL;
285 }
286 EXPORT_SYMBOL(ath11k_pcic_get_user_msi_assignment);
287 
288 void ath11k_pcic_get_ce_msi_idx(struct ath11k_base *ab, u32 ce_id, u32 *msi_idx)
289 {
290 	u32 i, msi_data_idx;
291 
292 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
293 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
294 			continue;
295 
296 		if (ce_id == i)
297 			break;
298 
299 		msi_data_idx++;
300 	}
301 	*msi_idx = msi_data_idx;
302 }
303 EXPORT_SYMBOL(ath11k_pcic_get_ce_msi_idx);
304 
305 static void ath11k_pcic_free_ext_irq(struct ath11k_base *ab)
306 {
307 	int i, j;
308 
309 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
310 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
311 
312 		for (j = 0; j < irq_grp->num_irq; j++)
313 			free_irq(ab->irq_num[irq_grp->irqs[j]], irq_grp);
314 
315 		netif_napi_del(&irq_grp->napi);
316 	}
317 }
318 
319 void ath11k_pcic_free_irq(struct ath11k_base *ab)
320 {
321 	int i, irq_idx;
322 
323 	for (i = 0; i < ab->hw_params.ce_count; i++) {
324 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
325 			continue;
326 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
327 		free_irq(ab->irq_num[irq_idx], &ab->ce.ce_pipe[i]);
328 	}
329 
330 	ath11k_pcic_free_ext_irq(ab);
331 }
332 EXPORT_SYMBOL(ath11k_pcic_free_irq);
333 
334 static void ath11k_pcic_ce_irq_enable(struct ath11k_base *ab, u16 ce_id)
335 {
336 	u32 irq_idx;
337 
338 	/* In case of one MSI vector, we handle irq enable/disable in a
339 	 * uniform way since we only have one irq
340 	 */
341 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
342 		return;
343 
344 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
345 	enable_irq(ab->irq_num[irq_idx]);
346 }
347 
348 static void ath11k_pcic_ce_irq_disable(struct ath11k_base *ab, u16 ce_id)
349 {
350 	u32 irq_idx;
351 
352 	/* In case of one MSI vector, we handle irq enable/disable in a
353 	 * uniform way since we only have one irq
354 	 */
355 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
356 		return;
357 
358 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
359 	disable_irq_nosync(ab->irq_num[irq_idx]);
360 }
361 
362 static void ath11k_pcic_ce_irqs_disable(struct ath11k_base *ab)
363 {
364 	int i;
365 
366 	clear_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
367 
368 	for (i = 0; i < ab->hw_params.ce_count; i++) {
369 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
370 			continue;
371 		ath11k_pcic_ce_irq_disable(ab, i);
372 	}
373 }
374 
375 static void ath11k_pcic_sync_ce_irqs(struct ath11k_base *ab)
376 {
377 	int i;
378 	int irq_idx;
379 
380 	for (i = 0; i < ab->hw_params.ce_count; i++) {
381 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
382 			continue;
383 
384 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
385 		synchronize_irq(ab->irq_num[irq_idx]);
386 	}
387 }
388 
389 static void ath11k_pcic_ce_tasklet(struct tasklet_struct *t)
390 {
391 	struct ath11k_ce_pipe *ce_pipe = from_tasklet(ce_pipe, t, intr_tq);
392 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
393 
394 	ath11k_ce_per_engine_service(ce_pipe->ab, ce_pipe->pipe_num);
395 
396 	enable_irq(ce_pipe->ab->irq_num[irq_idx]);
397 }
398 
399 static irqreturn_t ath11k_pcic_ce_interrupt_handler(int irq, void *arg)
400 {
401 	struct ath11k_ce_pipe *ce_pipe = arg;
402 	struct ath11k_base *ab = ce_pipe->ab;
403 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
404 
405 	if (!test_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags))
406 		return IRQ_HANDLED;
407 
408 	/* last interrupt received for this CE */
409 	ce_pipe->timestamp = jiffies;
410 
411 	disable_irq_nosync(ab->irq_num[irq_idx]);
412 
413 	tasklet_schedule(&ce_pipe->intr_tq);
414 
415 	return IRQ_HANDLED;
416 }
417 
418 static void ath11k_pcic_ext_grp_disable(struct ath11k_ext_irq_grp *irq_grp)
419 {
420 	struct ath11k_base *ab = irq_grp->ab;
421 	int i;
422 
423 	/* In case of one MSI vector, we handle irq enable/disable
424 	 * in a uniform way since we only have one irq
425 	 */
426 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
427 		return;
428 
429 	for (i = 0; i < irq_grp->num_irq; i++)
430 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
431 }
432 
433 static void __ath11k_pcic_ext_irq_disable(struct ath11k_base *sc)
434 {
435 	int i;
436 
437 	clear_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &sc->dev_flags);
438 
439 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
440 		struct ath11k_ext_irq_grp *irq_grp = &sc->ext_irq_grp[i];
441 
442 		ath11k_pcic_ext_grp_disable(irq_grp);
443 
444 		if (irq_grp->napi_enabled) {
445 			napi_synchronize(&irq_grp->napi);
446 			napi_disable(&irq_grp->napi);
447 			irq_grp->napi_enabled = false;
448 		}
449 	}
450 }
451 
452 static void ath11k_pcic_ext_grp_enable(struct ath11k_ext_irq_grp *irq_grp)
453 {
454 	struct ath11k_base *ab = irq_grp->ab;
455 	int i;
456 
457 	/* In case of one MSI vector, we handle irq enable/disable in a
458 	 * uniform way since we only have one irq
459 	 */
460 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
461 		return;
462 
463 	for (i = 0; i < irq_grp->num_irq; i++)
464 		enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
465 }
466 
467 void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab)
468 {
469 	int i;
470 
471 	set_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags);
472 
473 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
474 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
475 
476 		if (!irq_grp->napi_enabled) {
477 			napi_enable(&irq_grp->napi);
478 			irq_grp->napi_enabled = true;
479 		}
480 		ath11k_pcic_ext_grp_enable(irq_grp);
481 	}
482 }
483 EXPORT_SYMBOL(ath11k_pcic_ext_irq_enable);
484 
485 static void ath11k_pcic_sync_ext_irqs(struct ath11k_base *ab)
486 {
487 	int i, j, irq_idx;
488 
489 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
490 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
491 
492 		for (j = 0; j < irq_grp->num_irq; j++) {
493 			irq_idx = irq_grp->irqs[j];
494 			synchronize_irq(ab->irq_num[irq_idx]);
495 		}
496 	}
497 }
498 
499 void ath11k_pcic_ext_irq_disable(struct ath11k_base *ab)
500 {
501 	__ath11k_pcic_ext_irq_disable(ab);
502 	ath11k_pcic_sync_ext_irqs(ab);
503 }
504 EXPORT_SYMBOL(ath11k_pcic_ext_irq_disable);
505 
506 static int ath11k_pcic_ext_grp_napi_poll(struct napi_struct *napi, int budget)
507 {
508 	struct ath11k_ext_irq_grp *irq_grp = container_of(napi,
509 						struct ath11k_ext_irq_grp,
510 						napi);
511 	struct ath11k_base *ab = irq_grp->ab;
512 	int work_done;
513 	int i;
514 
515 	work_done = ath11k_dp_service_srng(ab, irq_grp, budget);
516 	if (work_done < budget) {
517 		napi_complete_done(napi, work_done);
518 		for (i = 0; i < irq_grp->num_irq; i++)
519 			enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
520 	}
521 
522 	if (work_done > budget)
523 		work_done = budget;
524 
525 	return work_done;
526 }
527 
528 static irqreturn_t ath11k_pcic_ext_interrupt_handler(int irq, void *arg)
529 {
530 	struct ath11k_ext_irq_grp *irq_grp = arg;
531 	struct ath11k_base *ab = irq_grp->ab;
532 	int i;
533 
534 	if (!test_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags))
535 		return IRQ_HANDLED;
536 
537 	ath11k_dbg(irq_grp->ab, ATH11K_DBG_PCI, "ext irq %d\n", irq);
538 
539 	/* last interrupt received for this group */
540 	irq_grp->timestamp = jiffies;
541 
542 	for (i = 0; i < irq_grp->num_irq; i++)
543 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
544 
545 	napi_schedule(&irq_grp->napi);
546 
547 	return IRQ_HANDLED;
548 }
549 
550 static int
551 ath11k_pcic_get_msi_irq(struct ath11k_base *ab, unsigned int vector)
552 {
553 	return ab->pci.ops->get_msi_irq(ab, vector);
554 }
555 
556 static int ath11k_pcic_ext_irq_config(struct ath11k_base *ab)
557 {
558 	int i, j, ret, num_vectors = 0;
559 	u32 user_base_data = 0, base_vector = 0;
560 	unsigned long irq_flags;
561 
562 	ret = ath11k_pcic_get_user_msi_assignment(ab, "DP", &num_vectors,
563 						  &user_base_data,
564 						  &base_vector);
565 	if (ret < 0)
566 		return ret;
567 
568 	irq_flags = IRQF_SHARED;
569 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
570 		irq_flags |= IRQF_NOBALANCING;
571 
572 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
573 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
574 		u32 num_irq = 0;
575 
576 		irq_grp->ab = ab;
577 		irq_grp->grp_id = i;
578 		init_dummy_netdev(&irq_grp->napi_ndev);
579 		netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi,
580 			       ath11k_pcic_ext_grp_napi_poll);
581 
582 		if (ab->hw_params.ring_mask->tx[i] ||
583 		    ab->hw_params.ring_mask->rx[i] ||
584 		    ab->hw_params.ring_mask->rx_err[i] ||
585 		    ab->hw_params.ring_mask->rx_wbm_rel[i] ||
586 		    ab->hw_params.ring_mask->reo_status[i] ||
587 		    ab->hw_params.ring_mask->rxdma2host[i] ||
588 		    ab->hw_params.ring_mask->host2rxdma[i] ||
589 		    ab->hw_params.ring_mask->rx_mon_status[i]) {
590 			num_irq = 1;
591 		}
592 
593 		irq_grp->num_irq = num_irq;
594 		irq_grp->irqs[0] = ATH11K_PCI_IRQ_DP_OFFSET + i;
595 
596 		for (j = 0; j < irq_grp->num_irq; j++) {
597 			int irq_idx = irq_grp->irqs[j];
598 			int vector = (i % num_vectors) + base_vector;
599 			int irq = ath11k_pcic_get_msi_irq(ab, vector);
600 
601 			if (irq < 0)
602 				return irq;
603 
604 			ab->irq_num[irq_idx] = irq;
605 
606 			ath11k_dbg(ab, ATH11K_DBG_PCI,
607 				   "irq %d group %d\n", irq, i);
608 
609 			irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
610 			ret = request_irq(irq, ath11k_pcic_ext_interrupt_handler,
611 					  irq_flags, "DP_EXT_IRQ", irq_grp);
612 			if (ret) {
613 				ath11k_err(ab, "failed request irq %d: %d\n",
614 					   vector, ret);
615 				return ret;
616 			}
617 		}
618 		ath11k_pcic_ext_grp_disable(irq_grp);
619 	}
620 
621 	return 0;
622 }
623 
624 int ath11k_pcic_config_irq(struct ath11k_base *ab)
625 {
626 	struct ath11k_ce_pipe *ce_pipe;
627 	u32 msi_data_start;
628 	u32 msi_data_count, msi_data_idx;
629 	u32 msi_irq_start;
630 	unsigned int msi_data;
631 	int irq, i, ret, irq_idx;
632 	unsigned long irq_flags;
633 
634 	ret = ath11k_pcic_get_user_msi_assignment(ab, "CE", &msi_data_count,
635 						  &msi_data_start, &msi_irq_start);
636 	if (ret)
637 		return ret;
638 
639 	irq_flags = IRQF_SHARED;
640 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
641 		irq_flags |= IRQF_NOBALANCING;
642 
643 	/* Configure CE irqs */
644 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
645 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
646 			continue;
647 
648 		msi_data = (msi_data_idx % msi_data_count) + msi_irq_start;
649 		irq = ath11k_pcic_get_msi_irq(ab, msi_data);
650 		if (irq < 0)
651 			return irq;
652 
653 		ce_pipe = &ab->ce.ce_pipe[i];
654 
655 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
656 
657 		tasklet_setup(&ce_pipe->intr_tq, ath11k_pcic_ce_tasklet);
658 
659 		ret = request_irq(irq, ath11k_pcic_ce_interrupt_handler,
660 				  irq_flags, irq_name[irq_idx], ce_pipe);
661 		if (ret) {
662 			ath11k_err(ab, "failed to request irq %d: %d\n",
663 				   irq_idx, ret);
664 			return ret;
665 		}
666 
667 		ab->irq_num[irq_idx] = irq;
668 		msi_data_idx++;
669 
670 		ath11k_pcic_ce_irq_disable(ab, i);
671 	}
672 
673 	ret = ath11k_pcic_ext_irq_config(ab);
674 	if (ret)
675 		return ret;
676 
677 	return 0;
678 }
679 EXPORT_SYMBOL(ath11k_pcic_config_irq);
680 
681 void ath11k_pcic_ce_irqs_enable(struct ath11k_base *ab)
682 {
683 	int i;
684 
685 	set_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
686 
687 	for (i = 0; i < ab->hw_params.ce_count; i++) {
688 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
689 			continue;
690 		ath11k_pcic_ce_irq_enable(ab, i);
691 	}
692 }
693 EXPORT_SYMBOL(ath11k_pcic_ce_irqs_enable);
694 
695 static void ath11k_pcic_kill_tasklets(struct ath11k_base *ab)
696 {
697 	int i;
698 
699 	for (i = 0; i < ab->hw_params.ce_count; i++) {
700 		struct ath11k_ce_pipe *ce_pipe = &ab->ce.ce_pipe[i];
701 
702 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
703 			continue;
704 
705 		tasklet_kill(&ce_pipe->intr_tq);
706 	}
707 }
708 
709 void ath11k_pcic_ce_irq_disable_sync(struct ath11k_base *ab)
710 {
711 	ath11k_pcic_ce_irqs_disable(ab);
712 	ath11k_pcic_sync_ce_irqs(ab);
713 	ath11k_pcic_kill_tasklets(ab);
714 }
715 EXPORT_SYMBOL(ath11k_pcic_ce_irq_disable_sync);
716 
717 void ath11k_pcic_stop(struct ath11k_base *ab)
718 {
719 	ath11k_pcic_ce_irq_disable_sync(ab);
720 	ath11k_ce_cleanup_pipes(ab);
721 }
722 EXPORT_SYMBOL(ath11k_pcic_stop);
723 
724 int ath11k_pcic_start(struct ath11k_base *ab)
725 {
726 	set_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags);
727 
728 	ath11k_pcic_ce_irqs_enable(ab);
729 	ath11k_ce_rx_post_buf(ab);
730 
731 	return 0;
732 }
733 EXPORT_SYMBOL(ath11k_pcic_start);
734 
735 int ath11k_pcic_map_service_to_pipe(struct ath11k_base *ab, u16 service_id,
736 				    u8 *ul_pipe, u8 *dl_pipe)
737 {
738 	const struct service_to_pipe *entry;
739 	bool ul_set = false, dl_set = false;
740 	int i;
741 
742 	for (i = 0; i < ab->hw_params.svc_to_ce_map_len; i++) {
743 		entry = &ab->hw_params.svc_to_ce_map[i];
744 
745 		if (__le32_to_cpu(entry->service_id) != service_id)
746 			continue;
747 
748 		switch (__le32_to_cpu(entry->pipedir)) {
749 		case PIPEDIR_NONE:
750 			break;
751 		case PIPEDIR_IN:
752 			WARN_ON(dl_set);
753 			*dl_pipe = __le32_to_cpu(entry->pipenum);
754 			dl_set = true;
755 			break;
756 		case PIPEDIR_OUT:
757 			WARN_ON(ul_set);
758 			*ul_pipe = __le32_to_cpu(entry->pipenum);
759 			ul_set = true;
760 			break;
761 		case PIPEDIR_INOUT:
762 			WARN_ON(dl_set);
763 			WARN_ON(ul_set);
764 			*dl_pipe = __le32_to_cpu(entry->pipenum);
765 			*ul_pipe = __le32_to_cpu(entry->pipenum);
766 			dl_set = true;
767 			ul_set = true;
768 			break;
769 		}
770 	}
771 
772 	if (WARN_ON(!ul_set || !dl_set))
773 		return -ENOENT;
774 
775 	return 0;
776 }
777 EXPORT_SYMBOL(ath11k_pcic_map_service_to_pipe);
778 
779 int ath11k_pcic_register_pci_ops(struct ath11k_base *ab,
780 				 const struct ath11k_pci_ops *pci_ops)
781 {
782 	if (!pci_ops)
783 		return 0;
784 
785 	/* Return error if mandatory pci_ops callbacks are missing */
786 	if (!pci_ops->get_msi_irq || !pci_ops->window_write32 ||
787 	    !pci_ops->window_read32)
788 		return -EINVAL;
789 
790 	ab->pci.ops = pci_ops;
791 	return 0;
792 }
793 EXPORT_SYMBOL(ath11k_pcic_register_pci_ops);
794 
795 void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
796 {
797 	int i;
798 
799 	for (i = 0; i < ab->hw_params.ce_count; i++) {
800 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
801 		    i == ATH11K_PCI_CE_WAKE_IRQ)
802 			continue;
803 		ath11k_pcic_ce_irq_enable(ab, i);
804 	}
805 }
806 EXPORT_SYMBOL(ath11k_pci_enable_ce_irqs_except_wake_irq);
807 
808 void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
809 {
810 	int i;
811 	int irq_idx;
812 	struct ath11k_ce_pipe *ce_pipe;
813 
814 	for (i = 0; i < ab->hw_params.ce_count; i++) {
815 		ce_pipe = &ab->ce.ce_pipe[i];
816 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
817 
818 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
819 		    i == ATH11K_PCI_CE_WAKE_IRQ)
820 			continue;
821 
822 		disable_irq_nosync(ab->irq_num[irq_idx]);
823 		synchronize_irq(ab->irq_num[irq_idx]);
824 		tasklet_kill(&ce_pipe->intr_tq);
825 	}
826 }
827 EXPORT_SYMBOL(ath11k_pci_disable_ce_irqs_except_wake_irq);
828