1 /*-
2  * Copyright (c) 2015-2016 Svatopluk Kraus
3  * Copyright (c) 2015-2016 Michal Meloun
5  * Copyright (c) 2015-2016 The FreeBSD Foundation
35  *	New-style Interrupt Framework
37  *  TODO: - add support for disconnected PICs.
38  *        - to support IPI (PPI) enabling on other CPUs if already started.
39  *        - to complete things for removable PICs.
172     "Number of IRQs");
201 	 * - 2 counters for each I/O interrupt.  in intr_irq_init()
202 	 * - mp_maxid + 1 counters for each IPI counters for SMP.  in intr_irq_init()
228 	snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s",  in intrcnt_setname()
229 	    INTRNAME_LEN - 1, name);  in intrcnt_setname()
241 	intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index);  in intrcnt_updatename()
251 	if (isrc->isrc_flags & INTR_ISRCF_PPI)  in isrc_increment_count()
252 		atomic_add_long(&isrc->isrc_count[0], 1);  in isrc_increment_count()
254 		isrc->isrc_count[0]++;  in isrc_increment_count()
264 	isrc->isrc_count[1]++;  in isrc_increment_straycount()
278 		snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name);  in isrc_update_name()
279 		intrcnt_setname(str, isrc->isrc_index);  in isrc_update_name()
280 		snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name,  in isrc_update_name()
282 		intrcnt_setname(str, isrc->isrc_index + 1);  in isrc_update_name()
284 		snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name);  in isrc_update_name()
285 		intrcnt_setname(str, isrc->isrc_index);  in isrc_update_name()
286 		snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name);  in isrc_update_name()
287 		intrcnt_setname(str, isrc->isrc_index + 1);  in isrc_update_name()
305 	if (index == -1)  in isrc_setup_counters()
308 	isrc->isrc_index = index;  in isrc_setup_counters()
309 	isrc->isrc_count = &intrcnt[index];  in isrc_setup_counters()
319 	int idx = isrc->isrc_index;  in isrc_release_counters()
341 	KASSERT(root->filter != NULL, ("%s: no filter", __func__));  in intr_irq_handler()
349 	oldframe = td->td_intr_frame;  in intr_irq_handler()
350 	td->td_intr_frame = tf;  in intr_irq_handler()
351 	(root->filter)(root->arg);  in intr_irq_handler()
352 	td->td_intr_frame = oldframe;  in intr_irq_handler()
356 	    (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN))  in intr_irq_handler()
368 	mtx_lock_spin(&parent->pic_child_lock);  in intr_child_irq_handler()
369 	SLIST_FOREACH(child, &parent->pic_children, pc_next) {  in intr_child_irq_handler()
370 		if (child->pc_start <= irq &&  in intr_child_irq_handler()
371 		    irq < (child->pc_start + child->pc_length)) {  in intr_child_irq_handler()
376 	mtx_unlock_spin(&parent->pic_child_lock);  in intr_child_irq_handler()
379 		return (child->pc_filter(child->pc_filter_arg, irq));  in intr_child_irq_handler()
395 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_dispatch()
399 	if (isrc->isrc_filter != NULL) {  in intr_isrc_dispatch()
401 		error = isrc->isrc_filter(isrc->isrc_arg, tf);  in intr_isrc_dispatch()
402 		PIC_POST_FILTER(isrc->isrc_dev, isrc);  in intr_isrc_dispatch()
407 	if (isrc->isrc_event != NULL) {  in intr_isrc_dispatch()
408 		if (intr_event_handle(isrc->isrc_event, tf) == 0)  in intr_isrc_dispatch()
412 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_dispatch()
450 	isrc->isrc_irq = irq;  in isrc_alloc_irq()
468 	if (isrc->isrc_irq >= intr_nirq)  in isrc_free_irq()
470 	if (irq_sources[isrc->isrc_irq] != isrc)  in isrc_free_irq()
473 	irq_sources[isrc->isrc_irq] = NULL;  in isrc_free_irq()
474 	isrc->isrc_irq = INTR_IRQ_INVALID;	/* just to be safe */  in isrc_free_irq()
507 	isrc->isrc_dev = dev;  in intr_isrc_register()
508 	isrc->isrc_irq = INTR_IRQ_INVALID;	/* just to be safe */  in intr_isrc_register()
509 	isrc->isrc_flags = flags;  in intr_isrc_register()
512 	vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap);  in intr_isrc_register()
526 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_register()
541 	if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0)  in intr_isrc_deregister()
562 	if (isrc->isrc_handlers == 0)  in intr_isrc_init_on_cpu()
564 	if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0)  in intr_isrc_init_on_cpu()
566 	if (isrc->isrc_flags & INTR_ISRCF_BOUND)  in intr_isrc_init_on_cpu()
567 		return (CPU_ISSET(cpu, &isrc->isrc_cpu));  in intr_isrc_init_on_cpu()
569 	CPU_SET(cpu, &isrc->isrc_cpu);  in intr_isrc_init_on_cpu()
591 	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {  in iscr_setup_filter()
595 	isrc->isrc_filter = filter;  in iscr_setup_filter()
596 	isrc->isrc_arg = arg;  in iscr_setup_filter()
613 	PIC_PRE_ITHREAD(isrc->isrc_dev, isrc);  in intr_isrc_pre_ithread()
624 	PIC_POST_ITHREAD(isrc->isrc_dev, isrc);  in intr_isrc_post_ithread()
635 	PIC_POST_FILTER(isrc->isrc_dev, isrc);  in intr_isrc_post_filter()
650 		CPU_ZERO(&isrc->isrc_cpu);  in intr_isrc_assign_cpu()
651 		isrc->isrc_flags &= ~INTR_ISRCF_BOUND;  in intr_isrc_assign_cpu()
653 		CPU_SETOF(cpu, &isrc->isrc_cpu);  in intr_isrc_assign_cpu()
654 		isrc->isrc_flags |= INTR_ISRCF_BOUND;  in intr_isrc_assign_cpu()
659 	 * re-balance it to another CPU or enable it on more CPUs. However,  in intr_isrc_assign_cpu()
664 		error = PIC_BIND_INTR(isrc->isrc_dev, isrc);  in intr_isrc_assign_cpu()
666 			CPU_ZERO(&isrc->isrc_cpu);  in intr_isrc_assign_cpu()
687 	error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq,
689 	    intr_isrc_assign_cpu, "%s:", isrc->isrc_name);
699 	if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) {
701 	if (isrc->isrc_event != NULL) {
705 		return (isrc->isrc_event != NULL ? EBUSY : 0);
707 	isrc->isrc_event = ie;
722 	ie = isrc->isrc_event;
723 	isrc->isrc_event = NULL;
740 	if (isrc->isrc_event == NULL) {
746 	error = intr_event_add_handler(isrc->isrc_event, name, filter, handler,
770 	/* Note that pic->pic_dev is never NULL on registered PIC. */
772 		if ((pic->pic_flags & FLAG_TYPE_MASK) !=
777 			if (xref == pic->pic_xref)
779 		} else if (xref == 0 || pic->pic_xref == 0) {
780 			if (dev == pic->pic_dev)
782 		} else if (xref == pic->pic_xref && dev == pic->pic_dev)
821 	pic->pic_xref = xref;
822 	pic->pic_dev = dev;
823 	pic->pic_flags = flags;
824 	mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN);
905 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC,
906 	    ("%s: Found a non-PIC controller: %s", __func__,
907 	     device_get_name(pic->pic_dev)));
922 	if (root->dev != NULL) {
927 	root->dev = dev;
928 	root->filter = filter;
929 	root->arg = arg;
936  * Add a handler to manage a sub range of a parents interrupts.
955 	newchild->pc_pic = pic;
956 	newchild->pc_filter = filter;
957 	newchild->pc_filter_arg = arg;
958 	newchild->pc_start = start;
959 	newchild->pc_length = length;
961 	mtx_lock_spin(&parent_pic->pic_child_lock);
963 	SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) {
964 		KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice",
968 	SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next);
969 	mtx_unlock_spin(&parent_pic->pic_child_lock);
985 	    (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC);
989 	switch (data->type) {
991 		KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
992 		    ("%s: Found a non-MSI controller: %s", __func__,
993 		     device_get_name(pic->pic_dev)));
995 		*isrc = msi->isrc;
999 		KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC,
1000 		    ("%s: Found a non-PIC controller: %s", __func__,
1001 		     device_get_name(pic->pic_dev)));
1002 		return (PIC_MAP_INTR(pic->pic_dev, data, isrc));
1016 		panic("Attempt to get isrc for non-active resource id: %u\n",
1018 	return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0);
1048 	return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data));
1065 		panic("Attempt to deactivate non-active resource id: %u\n",
1069 	error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data);
1118 		debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error,
1125 		debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name);
1131 	error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data);
1133 		isrc->isrc_handlers++;
1134 		if (isrc->isrc_handlers == 1)
1135 			PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
1156 	if (isrc == NULL || isrc->isrc_handlers == 0)
1162 	if (isrc->isrc_filter != NULL) {
1167 		isrc->isrc_filter = NULL;
1168 		isrc->isrc_arg = NULL;
1169 		isrc->isrc_handlers = 0;
1170 		PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
1171 		PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data);
1183 		isrc->isrc_handlers--;
1184 		if (isrc->isrc_handlers == 0)
1185 			PIC_DISABLE_INTR(isrc->isrc_dev, isrc);
1186 		PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data);
1206 	if (isrc == NULL || isrc->isrc_handlers == 0)
1209 	if (isrc->isrc_filter != NULL) {
1219 	error = intr_event_describe_handler(isrc->isrc_event, cookie, descr);
1240 	if (isrc == NULL || isrc->isrc_handlers == 0)
1243 	if (isrc->isrc_filter != NULL)
1246 	return (intr_event_bind(isrc->isrc_event, cpu));
1251  * For now just returns the next CPU according to round-robin.
1265 		return (CPU_FFS(cpumask) - 1);
1294 		if (isrc == NULL || isrc->isrc_handlers == 0 ||
1295 		    isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI))
1298 		if (isrc->isrc_event != NULL &&
1299 		    isrc->isrc_flags & INTR_ISRCF_BOUND &&
1300 		    isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1)
1303 		if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0)
1304 			CPU_ZERO(&isrc->isrc_cpu); /* start again */
1311 		if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0)
1312 			CPU_ZERO(&isrc->isrc_cpu);
1338 	data->type = type;
1339 	data->len = len;
1350  *  Register a MSI/MSI-X interrupt controller
1370     int maxcount, int *irqs)  argument
1383 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
1384 	    ("%s: Found a non-MSI controller: %s", __func__,
1385 	     device_get_name(pic->pic_dev)));
1389 	 * interrupt controller to map memory the msi source will need.
1391 	err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain);
1396 	err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc);
1403 		isrc[i]->isrc_iommu = domain;
1406 		msi-> isrc = isrc[i];
1408 		irqs[i] = intr_map_irq(pic->pic_dev, xref,
1418     int *irqs)  argument
1429 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
1430 	    ("%s: Found a non-MSI controller: %s", __func__,
1431 	     device_get_name(pic->pic_dev)));
1437 		    intr_map_get_map_data(irqs[i]);
1438 		KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI,
1439 		    ("%s: irq %d map data is not MSI", __func__,
1440 		    irqs[i]));
1441 		isrc[i] = msi->isrc;
1444 	MSI_IOMMU_DEINIT(pic->pic_dev, child);
1446 	err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc);
1450 			intr_unmap_irq(irqs[i]);
1471 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
1472 	    ("%s: Found a non-MSI controller: %s", __func__,
1473 	     device_get_name(pic->pic_dev)));
1477 	 * interrupt controller to map memory the msi source will need.
1479 	err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain);
1483 	err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc);
1487 	isrc->isrc_iommu = domain;
1490 	msi->isrc = isrc;
1491 	*irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi);
1507 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
1508 	    ("%s: Found a non-MSI controller: %s", __func__,
1509 	     device_get_name(pic->pic_dev)));
1513 	KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI,
1514 	    ("%s: irq %d map data is not MSI", __func__,
1516 	isrc = msi->isrc;
1522 	MSI_IOMMU_DEINIT(pic->pic_dev, child);
1524 	err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc);
1542 	KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI,
1543 	    ("%s: Found a non-MSI controller: %s", __func__,
1544 	     device_get_name(pic->pic_dev)));
1550 	err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data);
1553 	if (isrc->isrc_iommu != NULL)
1554 		iommu_translate_msi(isrc->isrc_iommu, addr);
1591 DB_SHOW_COMMAND_FLAGS(irqs, db_show_irqs, DB_CMD_MEMSAFE)  argument
1602 		num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0;
1603 		db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i,
1604 		    isrc->isrc_name, isrc->isrc_cpu.__bits[0],
1605 		    isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num);
1642 		isrc = irq_map[res_id]->isrc;
1654 		irq_map[res_id]->isrc = isrc;
1670 	data = irq_map[res_id]->map_data;
1689 	if (irq_map[res_id]->map_data != NULL)
1690 		len = irq_map[res_id]->map_data->len;
1701 		if (len != irq_map[res_id]->map_data->len)
1703 		memcpy(*data, irq_map[res_id]->map_data, len);
1705 	*map_dev = irq_map[res_id]->dev;
1706 	*map_xref = irq_map[res_id]->xref;
1722 	entry->dev = dev;
1723 	entry->xref = xref;
1724 	entry->map_data = data;
1725 	entry->isrc = NULL;
1765 	intr_free_intr_map_data(entry->map_data);
1787 	mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF);
1824 	if (index == -1)
1892 	isrc->isrc_handlers++;
1895 	KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
1897 	ii->ii_handler = hand;
1898 	ii->ii_handler_arg = arg;
1899 	ii->ii_isrc = isrc;
1900 	strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
1901 	ii->ii_count = intr_ipi_setup_counters(name);
1915 	if (ii->ii_count == NULL)
1931 	PIC_IPI_SEND(intr_ipi_dev, ii->ii_isrc, cpus, ipi);
1946 	if (ii->ii_count == NULL)
1949 	intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
1951 	ii->ii_handler(ii->ii_handler_arg);