Lines Matching +full:kind +full:- +full:of +full:- +full:intr
6 * Copyright (c) 2002-2005, K A Fraser
9 * Copyright © 2021-2023, Elliott Mitchell
14 * Permission is hereby granted, free of charge, to any person obtaining a copy
15 * of this source file (the "Software"), to deal in the Software without
17 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * all copies or substantial portions of the Software.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
55 #include <xen/xen-os.h>
60 #include <machine/xen/arch-intr.h>
67 * Per-cpu event channel processing state.
84 * A bitmap of ports that can be serviced from this CPU.
95 .last_processed_l1i = LONG_BIT - 1,
96 .last_processed_l2i = LONG_BIT - 1
107 * Modifying xen_intr_port_to_isrc[], or isrc->xi_port (implies the former)
109 * `!xen_intr_port_to_isrc[i] || (xen_intr_port_to_isrc[i]->ix_port == i)`
110 * MUST be true for all values of i which are valid indicies of the array.
112 * Acquire/release operations for isrc->xi_refcount require this lock be held.
117 /*------------------------- Private Functions --------------------------------*/
156 * This API is used to manage the port<=>CPU binding of event
159 * \note This operation does not preclude reception of an event
170 xen_clear_bit(port, pcpu->evtchn_enabled); in evtchn_cpu_mask_port()
179 * This API is used to manage the port<=>CPU binding of event
193 xen_set_bit(port, pcpu->evtchn_enabled); in evtchn_cpu_unmask_port()
208 if (is_valid_evtchn(isrc->xi_port)) { in xen_intr_release_isrc()
209 evtchn_mask_port(isrc->xi_port); in xen_intr_release_isrc()
210 evtchn_clear_port(isrc->xi_port); in xen_intr_release_isrc()
213 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); in xen_intr_release_isrc()
214 evtchn_cpu_unmask_port(0, isrc->xi_port); in xen_intr_release_isrc()
216 if (isrc->xi_close != 0) { in xen_intr_release_isrc()
217 struct evtchn_close close = { .port = isrc->xi_port }; in xen_intr_release_isrc()
223 xen_intr_port_to_isrc[isrc->xi_port] = NULL; in xen_intr_release_isrc()
239 * \param type The event channel type of local_port.
273 isrc->xi_cookie = NULL; in xen_intr_bind_isrc()
274 isrc->xi_type = type; in xen_intr_bind_isrc()
275 isrc->xi_port = local_port; in xen_intr_bind_isrc()
276 isrc->xi_close = false; in xen_intr_bind_isrc()
277 isrc->xi_cpu = 0; in xen_intr_bind_isrc()
278 refcount_init(&isrc->xi_refcount, 1); in xen_intr_bind_isrc()
280 xen_intr_port_to_isrc[isrc->xi_port] = isrc; in xen_intr_bind_isrc()
297 * the caller is in charge of setting that up. in xen_intr_bind_isrc()
315 * Determine the event channel ports at the given section of the
320 * \param idx The index of the section of the event channel bitmap to
332 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0])); in xen_intr_active_ports()
333 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0])); in xen_intr_active_ports()
334 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending)); in xen_intr_active_ports()
335 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled)); in xen_intr_active_ports()
336 return (sh->evtchn_pending[idx] in xen_intr_active_ports()
337 & ~sh->evtchn_mask[idx] in xen_intr_active_ports()
338 & pcpu->evtchn_enabled[idx]); in xen_intr_active_ports()
349 struct trapframe *trap_frame = curthread->td_intr_frame; in xen_intr_handle_upcall()
360 * increase td_intr_nesting_level ahead of dispatching the upcall in xen_intr_handle_upcall()
363 KASSERT(curthread->td_intr_nesting_level > 0, in xen_intr_handle_upcall()
377 v->evtchn_upcall_pending = 0; in xen_intr_handle_upcall()
378 /* No need for a barrier on x86 -- XCHG is a barrier on x86. */ in xen_intr_handle_upcall()
383 l1 = atomic_readandclear_xen_ulong(&v->evtchn_pending_sel); in xen_intr_handle_upcall()
385 l1i = pc->last_processed_l1i; in xen_intr_handle_upcall()
386 l2i = pc->last_processed_l2i; in xen_intr_handle_upcall()
397 l1i = LONG_BIT - 1; in xen_intr_handle_upcall()
398 l2i = LONG_BIT - 1; in xen_intr_handle_upcall()
401 l1i = ffsl(masked_l1) - 1; in xen_intr_handle_upcall()
411 l2i = LONG_BIT - 1; in xen_intr_handle_upcall()
414 l2i = ffsl(masked_l2) - 1; in xen_intr_handle_upcall()
425 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), in xen_intr_handle_upcall()
427 PCPU_GET(cpuid), isrc->xi_cpu)); in xen_intr_handle_upcall()
430 * Reduce interrupt nesting level ahead of calling the in xen_intr_handle_upcall()
431 * per-arch interrupt dispatch helper. This is in xen_intr_handle_upcall()
432 * required because the per-arch dispatcher will also in xen_intr_handle_upcall()
437 curthread->td_intr_nesting_level--; in xen_intr_handle_upcall()
439 curthread->td_intr_nesting_level++; in xen_intr_handle_upcall()
445 pc->last_processed_l1i = l1i; in xen_intr_handle_upcall()
446 pc->last_processed_l2i = l2i; in xen_intr_handle_upcall()
448 } while (l2i != LONG_BIT - 1); in xen_intr_handle_upcall()
475 _Static_assert(is_valid_evtchn(NR_EVENT_CHANNELS - 1), in xen_intr_init()
483 _Static_assert(!is_valid_evtchn(-1), in xen_intr_init()
484 "is_valid_evtchn(-1) fails (negative are invalid)"); in xen_intr_init()
486 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); in xen_intr_init()
489 * Set the per-cpu mask of CPU#0 to enable all, since by default all in xen_intr_init()
494 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, in xen_intr_init()
495 sizeof(pcpu->evtchn_enabled)); in xen_intr_init()
498 for (i = 0; i < nitems(s->evtchn_mask); i++) in xen_intr_init()
499 atomic_store_rel_xen_ulong(&s->evtchn_mask[i], ~0); in xen_intr_init()
508 /*--------------------------- Common PIC Functions ---------------------------*/
514 u_int cpu = isrc->xi_cpu; in xen_rebind_ipi()
524 isrc->xi_port = bind_ipi.port; in xen_rebind_ipi()
533 u_int cpu = isrc->xi_cpu; in xen_rebind_virq()
536 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, in xen_rebind_virq()
542 panic("unable to rebind xen VIRQ#%u: %d", isrc->xi_virq, error); in xen_rebind_virq()
544 isrc->xi_port = bind_virq.port; in xen_rebind_virq()
551 u_int cpu = isrc->xi_cpu; in xen_intr_rebind_isrc()
556 switch (isrc->xi_type) { in xen_intr_rebind_isrc()
567 prev = xen_intr_port_to_isrc[isrc->xi_port]; in xen_intr_rebind_isrc()
568 xen_intr_port_to_isrc[isrc->xi_port] = isrc; in xen_intr_rebind_isrc()
571 isrc->xi_cpu = 0; in xen_intr_rebind_isrc()
575 __func__, isrc->xi_port, cpu, error); in xen_intr_rebind_isrc()
578 evtchn_unmask_port(isrc->xi_port); in xen_intr_rebind_isrc()
593 /* Reset the per-CPU masks */ in xen_intr_resume()
598 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, in xen_intr_resume()
599 sizeof(pcpu->evtchn_enabled)); in xen_intr_resume()
603 for (i = 0; i < nitems(s->evtchn_mask); i++) in xen_intr_resume()
604 atomic_store_rel_xen_ulong(&s->evtchn_mask[i], ~0); in xen_intr_resume()
609 xen_intr_port_to_isrc[isrc_idx]->xi_port = in xen_intr_resume()
612 /* Remap in-use isrcs, using xen_intr_port_to_isrc as listing */ in xen_intr_resume()
616 /* empty or entry already taken care of */ in xen_intr_resume()
617 if (cur == NULL || cur->xi_port == isrc_idx) in xen_intr_resume()
623 KASSERT(!is_valid_evtchn(cur->xi_port), in xen_intr_resume()
624 ("%s(): Multiple channels on single intr?", in xen_intr_resume()
641 if (__predict_true(is_valid_evtchn(isrc->xi_port))) in xen_intr_disable_intr()
642 evtchn_mask_port(isrc->xi_port); in xen_intr_disable_intr()
649 * \param to_cpu The id of the CPU for handling future events.
665 if (!is_valid_evtchn(isrc->xi_port)) { in xen_intr_assign_cpu()
672 * delivery with an inconsistent state in isrc->xi_cpu. in xen_intr_assign_cpu()
674 masked = evtchn_test_and_set_mask(isrc->xi_port); in xen_intr_assign_cpu()
675 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || in xen_intr_assign_cpu()
676 (isrc->xi_type == EVTCHN_TYPE_IPI)) { in xen_intr_assign_cpu()
680 * all we need to do is update the per-CPU masks. in xen_intr_assign_cpu()
682 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); in xen_intr_assign_cpu()
683 isrc->xi_cpu = to_cpu; in xen_intr_assign_cpu()
684 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); in xen_intr_assign_cpu()
688 bind_vcpu.port = isrc->xi_port; in xen_intr_assign_cpu()
692 if (isrc->xi_cpu != to_cpu) { in xen_intr_assign_cpu()
695 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); in xen_intr_assign_cpu()
696 isrc->xi_cpu = to_cpu; in xen_intr_assign_cpu()
697 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); in xen_intr_assign_cpu()
703 evtchn_unmask_port(isrc->xi_port); in xen_intr_assign_cpu()
711 /*------------------- Virtual Interrupt Source PIC Functions -----------------*/
723 * needed because the event channel user-space device in xen_intr_disable_source()
724 * masks event channels on its filter as part of its in xen_intr_disable_source()
729 if (__predict_true(is_valid_evtchn(isrc->xi_port))) in xen_intr_disable_source()
730 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port); in xen_intr_disable_source()
742 if (isrc->xi_masked == 0) in xen_intr_enable_source()
743 evtchn_unmask_port(isrc->xi_port); in xen_intr_enable_source()
755 evtchn_unmask_port(isrc->xi_port); in xen_intr_enable_intr()
758 /*--------------------------- Public Functions -------------------------------*/
759 /*------- API comments for these methods can be found in xen/xenintr.h -------*/
778 isrc->xi_close = 0; in xen_intr_bind_local_port()
800 return (-error); in xen_intr_alloc_and_bind_local_port()
813 isrc->xi_close = 1; in xen_intr_alloc_and_bind_local_port()
835 return (-error); in xen_intr_bind_remote_port()
852 isrc->xi_close = 1; in xen_intr_bind_remote_port()
873 return (-error); in xen_intr_bind_virq()
895 if (isrc->xi_cpu != cpu) { in xen_intr_bind_virq()
910 isrc->xi_close = 1; in xen_intr_bind_virq()
911 isrc->xi_virq = virq; in xen_intr_bind_virq()
924 /* Same size as the one used by intr_handler->ih_name. */ in xen_intr_alloc_and_bind_ipi()
935 return (-error); in xen_intr_alloc_and_bind_ipi()
951 if (isrc->xi_cpu != cpu) { in xen_intr_alloc_and_bind_ipi()
965 isrc->xi_close = 1; in xen_intr_alloc_and_bind_ipi()
986 return (xen_arch_intr_describe(isrc, isrc->xi_cookie, descr)); in xen_intr_describe()
1003 if (refcount_release(&isrc->xi_refcount) == 0) { in xen_intr_unbind()
1009 if (isrc->xi_cookie != NULL) in xen_intr_unbind()
1010 xen_arch_intr_remove_handler(isrc, isrc->xi_cookie); in xen_intr_unbind()
1021 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || in xen_intr_signal()
1022 isrc->xi_type == EVTCHN_TYPE_IPI, in xen_intr_signal()
1024 struct evtchn_send send = { .port = isrc->xi_port }; in xen_intr_signal()
1038 return (isrc->xi_port); in xen_intr_port()
1050 if (isrc == NULL || isrc->xi_cookie != NULL) in xen_intr_add_handler()
1054 flags | INTR_EXCL, isrc, &isrc->xi_cookie); in xen_intr_add_handler()
1078 refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount); in xen_intr_get_evtchn_from_port()
1112 isrc->xi_port, xen_intr_print_type(isrc->xi_type)); in xen_intr_dump_port()
1113 if (isrc->xi_type == EVTCHN_TYPE_VIRQ) in xen_intr_dump_port()
1114 db_printf("\tVirq: %u\n", isrc->xi_virq); in xen_intr_dump_port()
1117 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]), in xen_intr_dump_port()
1118 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0])); in xen_intr_dump_port()
1120 db_printf("\tPer-CPU Masks: "); in xen_intr_dump_port()
1124 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled)); in xen_intr_dump_port()