183 static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc);
236 intrcnt_updatename(struct intr_irqsrc *isrc)  in intrcnt_updatename()  argument
241 	intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index);  in intrcnt_updatename()
248 isrc_increment_count(struct intr_irqsrc *isrc)  in isrc_increment_count()  argument
251 	if (isrc->isrc_flags & INTR_ISRCF_PPI)  in isrc_increment_count()
252 		atomic_add_long(&isrc->isrc_count[0], 1);  in isrc_increment_count()
254 		isrc->isrc_count[0]++;  in isrc_increment_count()
261 isrc_increment_straycount(struct intr_irqsrc *isrc)  in isrc_increment_straycount()  argument
264 	isrc->isrc_count[1]++;  in isrc_increment_straycount()
271 isrc_update_name(struct intr_irqsrc *isrc, const char *name)  in isrc_update_name()  argument
278 		snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name);  in isrc_update_name()
279 		intrcnt_setname(str, isrc->isrc_index);  in isrc_update_name()
280 		snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name,  in isrc_update_name()
282 		intrcnt_setname(str, isrc->isrc_index + 1);  in isrc_update_name()
284 		snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name);  in isrc_update_name()
285 		intrcnt_setname(str, isrc->isrc_index);  in isrc_update_name()
286 		snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name);  in isrc_update_name()
287 		intrcnt_setname(str, isrc->isrc_index + 1);  in isrc_update_name()
295 isrc_setup_counters(struct intr_irqsrc *isrc)  in isrc_setup_counters()  argument
308 	isrc->isrc_index = index;  in isrc_setup_counters()
309 	isrc->isrc_count = &intrcnt[index];  in isrc_setup_counters()
310 	isrc_update_name(isrc, NULL);  in isrc_setup_counters()
317 isrc_release_counters(struct intr_irqsrc *isrc)  in isrc_release_counters()  argument
319 	int idx = isrc->isrc_index;  in isrc_release_counters()
390 intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf)  in intr_isrc_dispatch()  argument
393 	KASSERT(isrc != NULL, ("%s: no source", __func__));  in intr_isrc_dispatch()
395 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_dispatch()
396 		isrc_increment_count(isrc);  in intr_isrc_dispatch()
399 	if (isrc->isrc_filter != NULL) {  in intr_isrc_dispatch()
401 		error = isrc->isrc_filter(isrc->isrc_arg, tf);  in intr_isrc_dispatch()
402 		PIC_POST_FILTER(isrc->isrc_dev, isrc);  in intr_isrc_dispatch()
407 	if (isrc->isrc_event != NULL) {  in intr_isrc_dispatch()
408 		if (intr_event_handle(isrc->isrc_event, tf) == 0)  in intr_isrc_dispatch()
412 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_dispatch()
413 		isrc_increment_straycount(isrc);  in intr_isrc_dispatch()
428 isrc_alloc_irq(struct intr_irqsrc *isrc)  in isrc_alloc_irq()  argument
450 	isrc->isrc_irq = irq;  in isrc_alloc_irq()
451 	irq_sources[irq] = isrc;  in isrc_alloc_irq()
463 isrc_free_irq(struct intr_irqsrc *isrc)  in isrc_free_irq()  argument
468 	if (isrc->isrc_irq >= intr_nirq)  in isrc_free_irq()
470 	if (irq_sources[isrc->isrc_irq] != isrc)  in isrc_free_irq()
473 	irq_sources[isrc->isrc_irq] = NULL;  in isrc_free_irq()
474 	isrc->isrc_irq = INTR_IRQ_INVALID;	/* just to be safe */  in isrc_free_irq()
500 intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags,  in intr_isrc_register()  argument
506 	bzero(isrc, sizeof(struct intr_irqsrc));  in intr_isrc_register()
507 	isrc->isrc_dev = dev;  in intr_isrc_register()
508 	isrc->isrc_irq = INTR_IRQ_INVALID;	/* just to be safe */  in intr_isrc_register()
509 	isrc->isrc_flags = flags;  in intr_isrc_register()
512 	vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap);  in intr_isrc_register()
516 	error = isrc_alloc_irq(isrc);  in intr_isrc_register()
526 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_register()
527 		isrc_setup_counters(isrc);  in intr_isrc_register()
536 intr_isrc_deregister(struct intr_irqsrc *isrc)  in intr_isrc_deregister()  argument
541 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_deregister()
542 		isrc_release_counters(isrc);  in intr_isrc_deregister()
543 	error = isrc_free_irq(isrc);  in intr_isrc_deregister()
559 intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu)  in intr_isrc_init_on_cpu()  argument
562 	if (isrc->isrc_handlers == 0)  in intr_isrc_init_on_cpu()
564 	if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0)  in intr_isrc_init_on_cpu()
566 	if (isrc->isrc_flags & INTR_ISRCF_BOUND)  in intr_isrc_init_on_cpu()
567 		return (CPU_ISSET(cpu, &isrc->isrc_cpu));  in intr_isrc_init_on_cpu()
569 	CPU_SET(cpu, &isrc->isrc_cpu);  in intr_isrc_init_on_cpu()
579 iscr_setup_filter(struct intr_irqsrc *isrc, const char *name,  in iscr_setup_filter()  argument
591 	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {  in iscr_setup_filter()
595 	isrc->isrc_filter = filter;  in iscr_setup_filter()
596 	isrc->isrc_arg = arg;  in iscr_setup_filter()
597 	isrc_update_name(isrc, name);  in iscr_setup_filter()
600 	*cookiep = isrc;  in iscr_setup_filter()
611 	struct intr_irqsrc *isrc = arg;  in intr_isrc_pre_ithread()  local
613 	PIC_PRE_ITHREAD(isrc->isrc_dev, isrc);  in intr_isrc_pre_ithread()
622 	struct intr_irqsrc *isrc = arg;  in intr_isrc_post_ithread()  local
624 	PIC_POST_ITHREAD(isrc->isrc_dev, isrc);  in intr_isrc_post_ithread()
633 	struct intr_irqsrc *isrc = arg;  in intr_isrc_post_filter()  local
635 	PIC_POST_FILTER(isrc->isrc_dev, isrc);  in intr_isrc_post_filter()
645 	struct intr_irqsrc *isrc = arg;  in intr_isrc_assign_cpu()  local
650 		CPU_ZERO(&isrc->isrc_cpu);  in intr_isrc_assign_cpu()
651 		isrc->isrc_flags &= ~INTR_ISRCF_BOUND;  in intr_isrc_assign_cpu()
653 		CPU_SETOF(cpu, &isrc->isrc_cpu);  in intr_isrc_assign_cpu()
654 		isrc->isrc_flags |= INTR_ISRCF_BOUND;  in intr_isrc_assign_cpu()
664 		error = PIC_BIND_INTR(isrc->isrc_dev, isrc);  in intr_isrc_assign_cpu()
666 			CPU_ZERO(&isrc->isrc_cpu);  in intr_isrc_assign_cpu()
682 isrc_event_create(struct intr_irqsrc *isrc)  argument
687 	error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq,
689 	    intr_isrc_assign_cpu, "%s:", isrc->isrc_name);
699 	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {
701 	if (isrc->isrc_event != NULL) {
705 		return (isrc->isrc_event != NULL ? EBUSY : 0);
707 	isrc->isrc_event = ie;
717 isrc_event_destroy(struct intr_irqsrc *isrc)  argument
722 	ie = isrc->isrc_event;
723 	isrc->isrc_event = NULL;
734 isrc_add_handler(struct intr_irqsrc *isrc, const char *name,  argument
740 	if (isrc->isrc_event == NULL) {
741 		error = isrc_event_create(isrc);
746 	error = intr_event_add_handler(isrc->isrc_event, name, filter, handler,
750 		intrcnt_updatename(isrc);
976     struct intr_irqsrc **isrc)  argument
995 		*isrc = msi->isrc;
1002 		return (PIC_MAP_INTR(pic->pic_dev, data, isrc));
1010 	struct intr_irqsrc *isrc;  local
1013 	isrc = intr_map_get_isrc(res_id);
1015 	if (isrc == NULL)
1018 	return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0);
1027 	struct intr_irqsrc *isrc;  local
1039 	error = intr_resolve_irq(map_dev, map_xref, data, &isrc);
1046 	intr_map_set_isrc(res_id, isrc);
1048 	return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data));
1055 	struct intr_irqsrc *isrc;  local
1063 	isrc = intr_map_get_isrc(res_id);
1064 	if (isrc == NULL)
1069 	error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data);
1082 	struct intr_irqsrc *isrc;  local
1090 	isrc = intr_map_get_isrc(res_id);
1091 	if (isrc == NULL) {
1116 		error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt,
1118 		debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error,
1123 		error = isrc_add_handler(isrc, name, filt, hand, arg, flags,
1125 		debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name);
1131 	error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data);
1133 		isrc->isrc_handlers++;
1134 		if (isrc->isrc_handlers == 1)
1135 			PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
1148 	struct intr_irqsrc *isrc;  local
1155 	isrc = intr_map_get_isrc(res_id);
1156 	if (isrc == NULL || isrc->isrc_handlers == 0)
1162 	if (isrc->isrc_filter != NULL) {
1163 		if (isrc != cookie)
1167 		isrc->isrc_filter = NULL;
1168 		isrc->isrc_arg = NULL;
1169 		isrc->isrc_handlers = 0;
1170 		PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
1171 		PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data);
1172 		isrc_update_name(isrc, NULL);
1177 	if (isrc != intr_handler_source(cookie))
1183 		isrc->isrc_handlers--;
1184 		if (isrc->isrc_handlers == 0)
1185 			PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
1186 		PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data);
1187 		intrcnt_updatename(isrc);
1198 	struct intr_irqsrc *isrc;  local
1205 	isrc = intr_map_get_isrc(res_id);
1206 	if (isrc == NULL || isrc->isrc_handlers == 0)
1209 	if (isrc->isrc_filter != NULL) {
1210 		if (isrc != cookie)
1214 		isrc_update_name(isrc, descr);
1219 	error = intr_event_describe_handler(isrc->isrc_event, cookie, descr);
1222 		intrcnt_updatename(isrc);
1232 	struct intr_irqsrc *isrc;  local
1239 	isrc = intr_map_get_isrc(res_id);
1240 	if (isrc == NULL || isrc->isrc_handlers == 0)
1243 	if (isrc->isrc_filter != NULL)
1244 		return (intr_isrc_assign_cpu(isrc, cpu));
1246 	return (intr_event_bind(isrc->isrc_event, cpu));
1284 	struct intr_irqsrc *isrc;  local
1293 		isrc = irq_sources[i];
1294 		if (isrc == NULL || isrc->isrc_handlers == 0 ||
1295 		    isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI))
1298 		if (isrc->isrc_event != NULL &&
1299 		    isrc->isrc_flags & INTR_ISRCF_BOUND &&
1300 		    isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1)
1303 		if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0)
1304 			CPU_ZERO(&isrc->isrc_cpu); /* start again */
1311 		if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0)
1312 			CPU_ZERO(&isrc->isrc_cpu);
1373 	struct intr_irqsrc **isrc;  local
1395 	isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK);
1396 	err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc);
1398 		free(isrc, M_INTRNG);
1403 		isrc[i]->isrc_iommu = domain;
1406 		msi-> isrc = isrc[i];
1411 	free(isrc, M_INTRNG);
1420 	struct intr_irqsrc **isrc;  local
1433 	isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK);
1441 		isrc[i] = msi->isrc;
1446 	err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc);
1449 		if (isrc[i] != NULL)
1453 	free(isrc, M_INTRNG);
1461 	struct intr_irqsrc *isrc;  local
1483 	err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc);
1487 	isrc->isrc_iommu = domain;
1490 	msi->isrc = isrc;
1498 	struct intr_irqsrc *isrc;  local
1516 	isrc = msi->isrc;
1517 	if (isrc == NULL) {
1524 	err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc);
1534 	struct intr_irqsrc *isrc;  local
1546 	isrc = intr_map_get_isrc(irq);
1547 	if (isrc == NULL)
1550 	err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data);
1553 	if (isrc->isrc_iommu != NULL)
1554 		iommu_translate_msi(isrc->isrc_iommu, addr);
1595 	struct intr_irqsrc *isrc;  local
1598 		isrc = irq_sources[i];
1599 		if (isrc == NULL)
1602 		num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0;
1604 		    isrc->isrc_name, isrc->isrc_cpu.__bits[0],
1605 		    isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num);
1623 	struct intr_irqsrc 	*isrc;  member
1637 	struct intr_irqsrc *isrc;  local
1639 	isrc = NULL;
1642 		isrc = irq_map[res_id]->isrc;
1645 	return (isrc);
1649 intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc)  argument
1654 		irq_map[res_id]->isrc = isrc;
1725 	entry->isrc = NULL;
1874 	struct intr_irqsrc *isrc;  local
1888 	error = PIC_IPI_SETUP(intr_ipi_dev, ipi, &isrc);
1892 	isrc->isrc_handlers++;
1899 	ii->ii_isrc = isrc;
1903 	PIC_ENABLE_INTR(intr_ipi_dev, isrc);