i86xpv/os/evtchn.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 * evtchn.c
 *
 * Communication via hypervisor event channels.
 *
 * Copyright (c) 2002-2005, K A Fraser
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/* some parts derived from netbsd's hypervisor_machdep.c 1.2.2.2 */

/*
 *
 * Copyright (c) 2004 Christian Limpach.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. This section intentionally left blank.
 * 4. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Section 3 of the above license was updated in response to bug 6379571.
 */

#include <sys/types.h>
#include <sys/hypervisor.h>
#include <sys/machsystm.h>
#include <sys/mutex.h>
#include <sys/evtchn_impl.h>
#include <sys/ddi_impldefs.h>
#include <sys/avintr.h>
#include <sys/cpuvar.h>
#include <sys/smp_impldefs.h>
#include <sys/archsystm.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/promif.h>
#include <sys/debug.h>
#include <sys/psm.h>
#include <sys/privregs.h>
#include <sys/trap.h>
#include <sys/atomic.h>
#include <sys/cpu.h>
#include <sys/psw.h>
#include <sys/traptrace.h>
#include <sys/stack.h>
#include <sys/x_call.h>
#include <xen/public/physdev.h>

/*
 * This file manages our association between hypervisor event channels and
 * Solaris's IRQs.  This is a one-to-one mapping, with the exception of
 * IPI IRQs, for which there is one event channel per CPU participating
 * in the IPI, and the clock VIRQ which also has an event channel per cpu
 * and the IRQ for /dev/xen/evtchn. The IRQ types are:
 *
 * IRQT_VIRQ:
 *	The hypervisor's standard virtual IRQ, used for the clock timer, for
 *	example.  This code allows any cpu to bind to one of these, although
 *	some are treated specially (i.e. VIRQ_DEBUG).
 *	Event channel binding is done via EVTCHNOP_bind_virq.
 *
 * IRQT_PIRQ:
 *	These associate a physical IRQ with an event channel via
 *	EVTCHNOP_bind_pirq.
 *
 * IRQT_IPI:
 *	A cross-call IRQ. Maps to "ncpus" event channels, each of which is
 *	bound to exactly one of the vcpus.  We do not currently support
 *	unbinding of IPIs (since Solaris doesn't need it). Uses
 *	EVTCHNOP_bind_ipi.
 *
 * IRQT_EVTCHN:
 *	A "normal" binding to an event channel, typically used by the frontend
 *      drivers to bind to the their backend event channel.
 *
 * IRQT_DEV_EVTCHN:
 *	This is a one-time IRQ used by /dev/xen/evtchn. Unlike other IRQs, we
 *	have a one-IRQ to many-evtchn mapping. We only track evtchn->irq for
 *	these event channels, which are managed via ec_irq_add/rm_evtchn().
 *	We enforce that IRQT_DEV_EVTCHN's representative evtchn (->ii_evtchn)
 *	is zero, and make any calls to irq_evtchn() an error, to prevent
 *	accidentally attempting to use the illegal evtchn 0.
 *
 * Suspend/resume
 *
 *	During a suspend/resume cycle, we need to tear down the event channels.
 *	All other mapping data is kept. The drivers will remove their own event
 *	channels via xendev on receiving a DDI_SUSPEND.  This leaves us with
 *	the IPIs and VIRQs, which we handle in ec_suspend() and ec_resume()
 *	below.
 *
 * CPU binding
 *
 *	When an event channel is bound to a CPU, we set a bit in a mask present
 *	in the machcpu (evt_affinity) to indicate that this CPU can accept this
 *	event channel.  For both IPIs and VIRQs, this binding is fixed at
 *	allocation time and we never modify it.  All other event channels are
 *	bound via the PSM either as part of add_avintr(), or interrupt
 *	redistribution (xen_psm_dis/enable_intr()) as a result of CPU
 *	offline/online.
 *
 * Locking
 *
 *	Updates are done holding the ec_lock.  The xen_callback_handler()
 *	routine reads the mapping data in a lockless fashion.  Additionally
 *	suspend takes ec_lock to prevent update races during a suspend/resume
 *	cycle.  The IPI info is also examined without the lock; this is OK
 *	since we only ever change IPI info during initial setup and resume.
 */

#define	IRQ_IS_CPUPOKE(irq) (ipi_info[XC_CPUPOKE_PIL].mi_irq == (irq))

#define	EVTCHN_MASKED(ev) \
	(HYPERVISOR_shared_info->evtchn_mask[(ev) >> EVTCHN_SHIFT] & \
	(1ul << ((ev) & ((1ul << EVTCHN_SHIFT) - 1))))

static short evtchn_to_irq[NR_EVENT_CHANNELS];
static cpuset_t evtchn_cpus[NR_EVENT_CHANNELS];
static int	evtchn_owner[NR_EVENT_CHANNELS];
#ifdef DEBUG
static kthread_t *evtchn_owner_thread[NR_EVENT_CHANNELS];
#endif

static irq_info_t irq_info[NR_IRQS];
static mec_info_t ipi_info[MAXIPL];
static mec_info_t virq_info[NR_VIRQS];
/*
 * Mailbox for communication with the evtchn device driver.
 * We rely on only cpu 0 servicing the event channels associated
 * with the driver.  i.e. all evtchn driver evtchns are bound to cpu 0.
 */
volatile int ec_dev_mbox;	/* mailbox for evtchn device driver */

/*
 * See the locking description above.
 */
kmutex_t ec_lock;

/*
 * Bitmap indicating which PIRQs require the hypervisor to be notified
 * on unmask.
 */
static unsigned long pirq_needs_eoi[NR_PIRQS / (sizeof (unsigned long) * NBBY)];

static int ec_debug_irq = INVALID_IRQ;
int ec_dev_irq = INVALID_IRQ;

int
xen_bind_virq(unsigned int virq, processorid_t cpu, int *port)
{
	evtchn_bind_virq_t bind;
	int err;

	bind.virq = virq;
	bind.vcpu = cpu;
	if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind)) == 0)
		*port = bind.port;
	else
		err = xen_xlate_errcode(err);
	return (err);
}

int
xen_bind_interdomain(int domid, int remote_port, int *port)
{
	evtchn_bind_interdomain_t bind;
	int err;

	bind.remote_dom  = domid;
	bind.remote_port = remote_port;
	if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
	    &bind)) == 0)
		*port = bind.local_port;
	else
		err = xen_xlate_errcode(err);
	return (err);
}

int
xen_alloc_unbound_evtchn(int domid, int *evtchnp)
{
	evtchn_alloc_unbound_t alloc;
	int err;

	alloc.dom = DOMID_SELF;
	alloc.remote_dom = domid;

	if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
	    &alloc)) == 0) {
		*evtchnp = alloc.port;
		/* ensure evtchn is masked till we're ready to use it */
		(void) ec_mask_evtchn(*evtchnp);
	} else {
		err = xen_xlate_errcode(err);
	}

	return (err);
}

static int
xen_close_evtchn(int evtchn)
{
	evtchn_close_t close;
	int err;

	close.port = evtchn;
	err = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
	if (err)
		err = xen_xlate_errcode(err);
	return (err);
}

static int
xen_bind_ipi(processorid_t cpu)
{
	evtchn_bind_ipi_t bind;

	ASSERT(MUTEX_HELD(&ec_lock));

	bind.vcpu = cpu;
	if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind) != 0)
		panic("xen_bind_ipi() failed");
	return (bind.port);
}

/* Send future instances of this interrupt to other vcpu. */
static void
xen_bind_vcpu(int evtchn, int cpu)
{
	evtchn_bind_vcpu_t bind;

	ASSERT(MUTEX_HELD(&ec_lock));

	bind.port = evtchn;
	bind.vcpu = cpu;
	if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind) != 0)
		panic("xen_bind_vcpu() failed");
}

static int
xen_bind_pirq(int pirq)
{
	evtchn_bind_pirq_t bind;
	int ret;

	bind.pirq = pirq;
	bind.flags = BIND_PIRQ__WILL_SHARE;
	if ((ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind)) != 0)
		panic("xen_bind_pirq() failed (err %d)", ret);
	return (bind.port);
}

/* unmask an evtchn and send upcall to appropriate vcpu if pending bit is set */
static void
xen_evtchn_unmask(int evtchn)
{
	evtchn_unmask_t unmask;

	unmask.port = evtchn;
	if (HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask) != 0)
		panic("xen_evtchn_unmask() failed");
}

static void
update_evtchn_affinity(int evtchn)
{
	cpu_t *cp;
	struct xen_evt_data *cpe;

	ASSERT(evtchn_to_irq[evtchn] != INVALID_IRQ);
	ASSERT(MUTEX_HELD(&ec_lock));

	/*
	 * Use lockless search of cpu_list, similar to mutex_vector_enter().
	 */
	kpreempt_disable();
	cp = cpu_list;
	do {
		cpe = cp->cpu_m.mcpu_evt_pend;
		if (CPU_IN_SET(evtchn_cpus[evtchn], cp->cpu_id))
			SET_EVTCHN_BIT(evtchn, cpe->evt_affinity);
		else
			CLEAR_EVTCHN_BIT(evtchn, cpe->evt_affinity);
	} while ((cp = cp->cpu_next) != cpu_list);
	kpreempt_enable();
}

static void
bind_evtchn_to_cpuset(int evtchn, cpuset_t cpus)
{
	ASSERT(evtchn_to_irq[evtchn] != INVALID_IRQ);

	CPUSET_ZERO(evtchn_cpus[evtchn]);
	CPUSET_OR(evtchn_cpus[evtchn], cpus);
	update_evtchn_affinity(evtchn);
}

static void
clear_evtchn_affinity(int evtchn)
{
	CPUSET_ZERO(evtchn_cpus[evtchn]);
	update_evtchn_affinity(evtchn);
}

static void
alloc_irq_evtchn(int irq, int index, int evtchn, int cpu)
{
	irq_info_t *irqp = &irq_info[irq];

	switch (irqp->ii_type) {
	case IRQT_IPI:
		ipi_info[index].mi_evtchns[cpu] = evtchn;
		irqp->ii_u.index = index;
		break;
	case IRQT_VIRQ:
		virq_info[index].mi_evtchns[cpu] = evtchn;
		irqp->ii_u.index = index;
		break;
	default:
		irqp->ii_u.evtchn = evtchn;
		break;
	}

	evtchn_to_irq[evtchn] = irq;

	/*
	 * If a CPU is not specified, we expect to bind it to a CPU later via
	 * the PSM.
	 */
	if (cpu != -1) {
		cpuset_t tcpus;
		CPUSET_ONLY(tcpus, cpu);
		bind_evtchn_to_cpuset(evtchn, tcpus);
	}
}

static int
alloc_irq(int type, int index, int evtchn, int cpu)
{
	int irq;
	irq_info_t *irqp;

	ASSERT(MUTEX_HELD(&ec_lock));
	ASSERT(type != IRQT_IPI || cpu != -1);

	for (irq = 0; irq < NR_IRQS; irq++) {
		if (irq_info[irq].ii_type == IRQT_UNBOUND)
			break;
	}

	if (irq == NR_IRQS)
		panic("No available IRQ to bind to: increase NR_IRQS!\n");

	irqp = &irq_info[irq];

	irqp->ii_type = type;
	/*
	 * Set irq/has_handler field to zero which means handler not installed
	 */
	irqp->ii_u2.has_handler = 0;

	alloc_irq_evtchn(irq, index, evtchn, cpu);
	return (irq);
}

static int
irq_evtchn(irq_info_t *irqp)
{
	int evtchn;

	ASSERT(irqp->ii_type != IRQT_DEV_EVTCHN);

	switch (irqp->ii_type) {
	case IRQT_IPI:
		ASSERT(irqp->ii_u.index != 0);
		evtchn = ipi_info[irqp->ii_u.index].mi_evtchns[CPU->cpu_id];
		break;
	case IRQT_VIRQ:
		evtchn = virq_info[irqp->ii_u.index].mi_evtchns[CPU->cpu_id];
		break;
	default:
		evtchn = irqp->ii_u.evtchn;
		break;
	}

	return (evtchn);
}

static void
unbind_evtchn(ushort_t *evtchnp)
{
	int err;

	ASSERT(MUTEX_HELD(&ec_lock));

	ASSERT(*evtchnp != 0);

	err = xen_close_evtchn(*evtchnp);
	ASSERT(err == 0);
	clear_evtchn_affinity(*evtchnp);
	evtchn_to_irq[*evtchnp] = INVALID_IRQ;
	*evtchnp = 0;
}

static void
pirq_unmask_notify(int pirq)
{
	struct physdev_eoi eoi;

	if (TEST_EVTCHN_BIT(pirq, &pirq_needs_eoi[0])) {
		eoi.irq = pirq;
		(void) HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
	}
}

static void
pirq_query_unmask(int pirq)
{
	struct physdev_irq_status_query irq_status;

	irq_status.irq = pirq;
	(void) HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status);
	CLEAR_EVTCHN_BIT(pirq, &pirq_needs_eoi[0]);
	if (irq_status.flags & XENIRQSTAT_needs_eoi)
		SET_EVTCHN_BIT(pirq, &pirq_needs_eoi[0]);
}

static void
end_pirq(int irq)
{
	int evtchn = irq_evtchn(&irq_info[irq]);

	ec_unmask_evtchn(evtchn);
	pirq_unmask_notify(IRQ_TO_PIRQ(irq));
}

/*
 * probe if a pirq is available to bind to, return 1 if available
 * else return 0.
 * Note that for debug versions of xen this probe may cause an in use IRQ
 * warning message from xen.
 */
int
ec_probe_pirq(int pirq)
{
	evtchn_bind_pirq_t bind;

	bind.pirq = pirq;
	bind.flags = 0;
	if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind) != 0) {
		return (0);
	} else {
		(void) xen_close_evtchn(bind.port);
		return (1);
	}
}

/*
 * Bind an event channel to a vcpu
 */
void
ec_bind_vcpu(int evtchn, int cpu)
{
	mutex_enter(&ec_lock);
	xen_bind_vcpu(evtchn, cpu);
	mutex_exit(&ec_lock);
}

/*
 * Set up a physical device irq to be associated with an event channel.
 */
void
ec_setup_pirq(int irq, int ipl, cpuset_t cpus)
{
	int evtchn;
	irq_info_t *irqp = &irq_info[irq];

	/*
	 * Test if this PIRQ is already bound to an evtchn,
	 * which means it is a shared IRQ and we don't want to
	 * bind and do some initial setup that has already been
	 * done for this irq on a previous trip through this code.
	 */
	if (irqp->ii_u.evtchn == INVALID_EVTCHN) {
		evtchn = xen_bind_pirq(irq);

		pirq_query_unmask(IRQ_TO_PIRQ(irq));

		irqp->ii_type = IRQT_PIRQ;
		irqp->ii_u.evtchn = evtchn;

		evtchn_to_irq[evtchn] = irq;
		irqp->ii_u2.ipl = ipl;
		ec_set_irq_affinity(irq, cpus);
		ec_enable_irq(irq);
		pirq_unmask_notify(IRQ_TO_PIRQ(irq));
	} else {
		ASSERT(irqp->ii_u2.ipl != 0);
		cmn_err(CE_NOTE, "IRQ%d is shared", irq);
		if (ipl > irqp->ii_u2.ipl)
			irqp->ii_u2.ipl = ipl;
	}
}

void
ec_unbind_irq(int irq)
{
	irq_info_t *irqp = &irq_info[irq];
	mec_info_t *virqp;
	int drop_lock = 0;
	int type, i;

	/*
	 * Nasty, but we need this during suspend.
	 */
	if (mutex_owner(&ec_lock) != curthread) {
		mutex_enter(&ec_lock);
		drop_lock = 1;
	}

	type = irqp->ii_type;

	ASSERT((type == IRQT_EVTCHN) || (type == IRQT_PIRQ) ||
	    (type == IRQT_VIRQ));

	if ((type == IRQT_EVTCHN) || (type == IRQT_PIRQ)) {
		/* There's only one event channel associated with this irq */
		unbind_evtchn(&irqp->ii_u.evtchn);
	} else if (type == IRQT_VIRQ) {
		/*
		 * Each cpu on the system can have it's own event channel
		 * associated with a virq.  Unbind them all.
		 */
		virqp = &virq_info[irqp->ii_u.index];
		for (i = 0; i < NCPU; i++) {
			if (virqp->mi_evtchns[i] != 0)
				unbind_evtchn(&virqp->mi_evtchns[i]);
		}
		/* Mark the virq structure as invalid. */
		virqp->mi_irq = INVALID_IRQ;
	}

	bzero(irqp, sizeof (*irqp));
	/* Re-reserve PIRQ. */
	if (type == IRQT_PIRQ)
		irqp->ii_type = IRQT_PIRQ;

	if (drop_lock)
		mutex_exit(&ec_lock);
}

/*ARGSUSED*/
static int
do_nothing_function(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3)
{
	return (0);
}

/*
 * Rebind an event channel for delivery to a CPU.
 */
void
ec_set_irq_affinity(int irq, cpuset_t dest)
{
	int evtchn, tcpu;
	irq_info_t *irqp = &irq_info[irq];

	mutex_enter(&ec_lock);

	ASSERT(irq < NR_IRQS);
	ASSERT(irqp->ii_type != IRQT_UNBOUND);

	/*
	 * Binding is done at allocation time for these types, so we should
	 * never modify them.
	 */
	if (irqp->ii_type == IRQT_IPI || irqp->ii_type == IRQT_VIRQ ||
	    irqp->ii_type == IRQT_DEV_EVTCHN) {
		mutex_exit(&ec_lock);
		return;
	}

	CPUSET_FIND(dest, tcpu);
	ASSERT(tcpu != CPUSET_NOTINSET);

	evtchn = irq_evtchn(irqp);

	xen_bind_vcpu(evtchn, tcpu);

	bind_evtchn_to_cpuset(evtchn, dest);

	mutex_exit(&ec_lock);

	/*
	 * Now send the new target processor a NOP IPI. When this returns,
	 * it will check for any pending interrupts, and so service any that
	 * got delivered to the wrong processor by mistake.
	 */
	xc_call(NULL, NULL, NULL, X_CALL_HIPRI, dest, do_nothing_function);
}

int
ec_set_irq_priority(int irq, int pri)
{
	irq_info_t *irqp;

	if (irq >= NR_IRQS)
		return (-1);

	irqp = &irq_info[irq];

	if (irqp->ii_type == IRQT_UNBOUND)
		return (-1);

	irqp->ii_u2.ipl = pri;

	return (0);
}

void
ec_clear_irq_priority(int irq)
{
	irq_info_t *irqp = &irq_info[irq];

	ASSERT(irq < NR_IRQS);
	ASSERT(irqp->ii_type != IRQT_UNBOUND);

	irqp->ii_u2.ipl = 0;
}

int
ec_bind_evtchn_to_irq(int evtchn)
{
	mutex_enter(&ec_lock);

	ASSERT(evtchn_to_irq[evtchn] == INVALID_IRQ);

	(void) alloc_irq(IRQT_EVTCHN, 0, evtchn, -1);

	mutex_exit(&ec_lock);
	return (evtchn_to_irq[evtchn]);
}

int
ec_bind_virq_to_irq(int virq, int cpu)
{
	int err;
	int evtchn;
	mec_info_t *virqp;

	virqp = &virq_info[virq];
	mutex_enter(&ec_lock);

	err = xen_bind_virq(virq, cpu, &evtchn);
	ASSERT(err == 0);

	ASSERT(evtchn_to_irq[evtchn] == INVALID_IRQ);

	if (virqp->mi_irq == INVALID_IRQ) {
		virqp->mi_irq = alloc_irq(IRQT_VIRQ, virq, evtchn, cpu);
	} else {
		alloc_irq_evtchn(virqp->mi_irq, virq, evtchn, cpu);
	}

	mutex_exit(&ec_lock);

	return (virqp->mi_irq);
}

int
ec_bind_ipi_to_irq(int ipl, int cpu)
{
	int evtchn;
	ulong_t flags;
	mec_info_t *ipip;

	mutex_enter(&ec_lock);

	ipip = &ipi_info[ipl];

	evtchn = xen_bind_ipi(cpu);

	ASSERT(evtchn_to_irq[evtchn] == INVALID_IRQ);

	if (ipip->mi_irq == INVALID_IRQ) {
		ipip->mi_irq = alloc_irq(IRQT_IPI, ipl, evtchn, cpu);
	} else {
		alloc_irq_evtchn(ipip->mi_irq, ipl, evtchn, cpu);
	}

	/*
	 * Unmask the new evtchn so that it can be seen by the target cpu
	 */
	flags = intr_clear();
	ec_unmask_evtchn(evtchn);
	intr_restore(flags);

	mutex_exit(&ec_lock);
	return (ipip->mi_irq);
}

/*
 * When bringing up a CPU, bind to all the IPIs that CPU0 bound.
 */
void
ec_bind_cpu_ipis(int cpu)
{
	int i;

	for (i = 0; i < MAXIPL; i++) {
		mec_info_t *ipip = &ipi_info[i];
		if (ipip->mi_irq == INVALID_IRQ)
			continue;

		(void) ec_bind_ipi_to_irq(i, cpu);
	}
}

/*
 * Can this IRQ be rebound to another CPU?
 */
int
ec_irq_rebindable(int irq)
{
	irq_info_t *irqp = &irq_info[irq];

	if (irqp->ii_u.evtchn == 0)
		return (0);

	return (irqp->ii_type == IRQT_EVTCHN || irqp->ii_type == IRQT_PIRQ);
}

/*
 * Should this IRQ be unbound from this CPU (which is being offlined) to
 * another?
 */
int
ec_irq_needs_rebind(int irq, int cpu)
{
	irq_info_t *irqp = &irq_info[irq];

	return (ec_irq_rebindable(irq) &&
	    CPU_IN_SET(evtchn_cpus[irqp->ii_u.evtchn], cpu));
}

void
ec_send_ipi(int ipl, int cpu)
{
	mec_info_t *ipip = &ipi_info[ipl];

	ASSERT(ipip->mi_irq != INVALID_IRQ);

	ec_notify_via_evtchn(ipip->mi_evtchns[cpu]);
}

void
ec_try_ipi(int ipl, int cpu)
{
	mec_info_t *ipip = &ipi_info[ipl];

	if (ipip->mi_irq == INVALID_IRQ || ipip->mi_irq == 0)
		return;

	ec_notify_via_evtchn(ipip->mi_evtchns[cpu]);
}

void
ec_irq_add_evtchn(int irq, int evtchn)
{
	mutex_enter(&ec_lock);

	/*
	 * See description of IRQT_DEV_EVTCHN above.
	 */
	ASSERT(irq == ec_dev_irq);

	alloc_irq_evtchn(irq, 0, evtchn, 0);
	/*
	 * We enforce that the representative event channel for IRQT_DEV_EVTCHN
	 * is zero, so PSM operations on it have no effect.
	 */
	irq_info[irq].ii_u.evtchn = 0;
	mutex_exit(&ec_lock);
}

void
ec_irq_rm_evtchn(int irq, int evtchn)
{
	ushort_t ec = evtchn;

	mutex_enter(&ec_lock);
	ASSERT(irq == ec_dev_irq);
	unbind_evtchn(&ec);
	mutex_exit(&ec_lock);
}

/*
 * Allocate an /dev/xen/evtchn IRQ.  See the big comment at the top
 * for an explanation.
 */
int
ec_dev_alloc_irq(void)
{
	int i;
	irq_info_t *irqp;

	for (i = 0; i < NR_IRQS; i++) {
		if (irq_info[i].ii_type == IRQT_UNBOUND)
			break;
	}

	ASSERT(i != NR_IRQS);

	irqp = &irq_info[i];
	irqp->ii_type = IRQT_DEV_EVTCHN;
	irqp->ii_u2.ipl = IPL_EVTCHN;
	/*
	 * Force the evtchn to zero for the special evtchn device irq
	 */
	irqp->ii_u.evtchn = 0;
	return (i);
}

void
ec_enable_irq(unsigned int irq)
{
	ulong_t flag;
	irq_info_t *irqp = &irq_info[irq];

	if (irqp->ii_type == IRQT_DEV_EVTCHN)
		return;

	flag = intr_clear();
	ec_unmask_evtchn(irq_evtchn(irqp));
	intr_restore(flag);
}

void
ec_disable_irq(unsigned int irq)
{
	irq_info_t *irqp = &irq_info[irq];

	if (irqp->ii_type == IRQT_DEV_EVTCHN)
		return;

	/*
	 * Spin till we are the one to mask the evtchn
	 * Ensures no one else can be servicing this evtchn.
	 */
	while (!ec_mask_evtchn(irq_evtchn(irqp)))
		SMT_PAUSE();
}

static int
ec_evtchn_pending(uint_t ev)
{
	uint_t evi;
	shared_info_t *si = HYPERVISOR_shared_info;

	evi = ev >> EVTCHN_SHIFT;
	ev &= (1ul << EVTCHN_SHIFT) - 1;
	return ((si->evtchn_pending[evi] & (1ul << ev)) != 0);
}

int
ec_pending_irq(unsigned int irq)
{
	int evtchn = irq_evtchn(&irq_info[irq]);

	return (ec_evtchn_pending(evtchn));
}

void
ec_clear_irq(int irq)
{
	irq_info_t *irqp = &irq_info[irq];
	int evtchn;

	if (irqp->ii_type == IRQT_DEV_EVTCHN)
		return;

	ASSERT(irqp->ii_type != IRQT_UNBOUND);

	evtchn = irq_evtchn(irqp);

	ASSERT(EVTCHN_MASKED(evtchn));
	ec_clear_evtchn(evtchn);
}

void
ec_unmask_irq(int irq)
{
	ulong_t flags;
	irq_info_t *irqp = &irq_info[irq];

	flags = intr_clear();
	switch (irqp->ii_type) {
	case IRQT_PIRQ:
		end_pirq(irq);
		break;
	case IRQT_DEV_EVTCHN:
		break;
	default:
		ec_unmask_evtchn(irq_evtchn(irqp));
		break;
	}
	intr_restore(flags);
}

void
ec_try_unmask_irq(int irq)
{
	ulong_t flags;
	irq_info_t *irqp = &irq_info[irq];
	int evtchn;

	flags = intr_clear();
	switch (irqp->ii_type) {
	case IRQT_PIRQ:
		end_pirq(irq);
		break;
	case IRQT_DEV_EVTCHN:
		break;
	default:
		if ((evtchn = irq_evtchn(irqp)) != 0)
			ec_unmask_evtchn(evtchn);
		break;
	}
	intr_restore(flags);
}

/*
 * Poll until an event channel is ready or 'check_func' returns true.  This can
 * only be used in a situation where interrupts are masked, otherwise we have a
 * classic time-of-check vs. time-of-use race.
 */
void
ec_wait_on_evtchn(int evtchn, int (*check_func)(void *), void *arg)
{
	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
		while (!check_func(arg))
			(void) HYPERVISOR_yield();
		return;
	}

	ASSERT(CPU->cpu_m.mcpu_vcpu_info->evtchn_upcall_mask != 0);

	for (;;) {
		evtchn_port_t ports[1];

		ports[0] = evtchn;

		ec_clear_evtchn(evtchn);

		if (check_func(arg))
			return;

		(void) HYPERVISOR_poll(ports, 1, 0);
	}
}

void
ec_wait_on_ipi(int ipl, int (*check_func)(void *), void *arg)
{
	mec_info_t *ipip = &ipi_info[ipl];

	if (ipip->mi_irq == INVALID_IRQ || ipip->mi_irq == 0)
		return;

	ec_wait_on_evtchn(ipip->mi_evtchns[CPU->cpu_id], check_func, arg);
}

void
ec_suspend(void)
{
	irq_info_t *irqp;
	ushort_t *evtchnp;
	int i;
	int c;

	ASSERT(MUTEX_HELD(&ec_lock));

	for (i = 0; i < MAXIPL; i++) {
		if (ipi_info[i].mi_irq == INVALID_IRQ)
			continue;

		for (c = 0; c < NCPU; c++) {
			if (cpu[c] == NULL)
				continue;

			if (CPU_IN_SET(cpu_suspend_lost_set, c))
				continue;

			evtchnp = &ipi_info[i].mi_evtchns[c];
			ASSERT(*evtchnp != 0);
			unbind_evtchn(evtchnp);
		}
	}

	for (i = 0; i < NR_VIRQS; i++) {
		if (virq_info[i].mi_irq == INVALID_IRQ)
			continue;

		/*
		 * If we're sharing a single event channel across all CPUs, we
		 * should only unbind once.
		 */
		if (virq_info[i].mi_shared) {
			evtchnp = &virq_info[i].mi_evtchns[0];
			unbind_evtchn(evtchnp);
			for (c = 1; c < NCPU; c++)
				virq_info[i].mi_evtchns[c] = 0;
		} else {
			for (c = 0; c < NCPU; c++) {
				if (cpu[c] == NULL)
					continue;

				evtchnp = &virq_info[i].mi_evtchns[c];
				if (*evtchnp != 0)
					unbind_evtchn(evtchnp);
			}
		}
	}

	for (i = 0; i < NR_IRQS; i++) {
		irqp = &irq_info[i];

		switch (irqp->ii_type) {
		case IRQT_EVTCHN:
		case IRQT_DEV_EVTCHN:
			(void) HYPERVISOR_shutdown(SHUTDOWN_crash);
			break;
		case IRQT_PIRQ:
			if (irqp->ii_u.evtchn != 0)
				(void) HYPERVISOR_shutdown(SHUTDOWN_crash);
			break;
		default:
			break;
		}
	}
}

/*
 * The debug irq is special, we only have one evtchn and irq but we allow all
 * cpus to service it.  It's marked as shared and we propogate the event
 * channel into all CPUs by hand.
 */
static void
share_virq(mec_info_t *virqp)
{
	int evtchn = virqp->mi_evtchns[0];
	cpuset_t tset;
	int i;

	ASSERT(evtchn != 0);

	virqp->mi_shared = 1;

	for (i = 1; i < NCPU; i++)
		virqp->mi_evtchns[i] = evtchn;
	CPUSET_ALL(tset);
	bind_evtchn_to_cpuset(evtchn, tset);
}

static void
virq_resume(int virq)
{
	mec_info_t *virqp = &virq_info[virq];
	int evtchn;
	int i, err;

	for (i = 0; i < NCPU; i++) {
		cpuset_t tcpus;

		if (cpu[i] == NULL || CPU_IN_SET(cpu_suspend_lost_set, i))
			continue;

		err = xen_bind_virq(virq, i, &evtchn);
		ASSERT(err == 0);

		virqp->mi_evtchns[i] = evtchn;
		evtchn_to_irq[evtchn] = virqp->mi_irq;
		CPUSET_ONLY(tcpus, i);
		bind_evtchn_to_cpuset(evtchn, tcpus);
		ec_unmask_evtchn(evtchn);
		/*
		 * only timer VIRQ is bound to all cpus
		 */
		if (virq != VIRQ_TIMER)
			break;
	}

	if (virqp->mi_shared)
		share_virq(virqp);
}

static void
ipi_resume(int ipl)
{
	mec_info_t *ipip = &ipi_info[ipl];
	int i;

	for (i = 0; i < NCPU; i++) {
		cpuset_t tcpus;
		int evtchn;

		if (cpu[i] == NULL || CPU_IN_SET(cpu_suspend_lost_set, i))
			continue;

		evtchn = xen_bind_ipi(i);
		ipip->mi_evtchns[i] = evtchn;
		evtchn_to_irq[evtchn] = ipip->mi_irq;
		CPUSET_ONLY(tcpus, i);
		bind_evtchn_to_cpuset(evtchn, tcpus);
		ec_unmask_evtchn(evtchn);
	}
}

void
ec_resume(void)
{
	int i;

	/* New event-channel space is not 'live' yet. */
	for (i = 0; i < NR_EVENT_CHANNELS; i++)
		(void) ec_mask_evtchn(i);

	for (i = 0; i < MAXIPL; i++) {
		if (ipi_info[i].mi_irq == INVALID_IRQ)
			continue;
		ipi_resume(i);
	}

	for (i = 0; i < NR_VIRQS; i++) {
		if (virq_info[i].mi_irq == INVALID_IRQ)
			continue;
		virq_resume(i);
	}
}

void
ec_init(void)
{
	int i;
	mutex_init(&ec_lock, NULL, MUTEX_SPIN, (void *)ipltospl(SPL7));

	for (i = 0; i < NR_EVENT_CHANNELS; i++) {
		CPUSET_ZERO(evtchn_cpus[i]);
		evtchn_to_irq[i] = INVALID_IRQ;
		(void) ec_mask_evtchn(i);
	}

	for (i = 0; i < MAXIPL; i++)
		ipi_info[i].mi_irq = INVALID_IRQ;

	for (i = 0; i < NR_VIRQS; i++)
		virq_info[i].mi_irq = INVALID_IRQ;

	/*
	 * Phys IRQ space is statically bound (1:1 mapping), grab the IRQs
	 * now.
	 */
	for (i = PIRQ_BASE; i < NR_PIRQS; i++) {
		irq_info[PIRQ_TO_IRQ(i)].ii_type = IRQT_PIRQ;
	}
}

void
ec_init_debug_irq()
{
	int irq;

	irq = ec_bind_virq_to_irq(VIRQ_DEBUG, 0);
	(void) add_avintr(NULL, IPL_DEBUG, (avfunc)xen_debug_handler,
	    "debug", irq, NULL, NULL, NULL, NULL);

	mutex_enter(&ec_lock);
	share_virq(&virq_info[irq_info[irq].ii_u.index]);
	mutex_exit(&ec_lock);
	ec_debug_irq = irq;
}

#define	UNBLOCKED_EVENTS(si, ix, cpe, cpu_id) \
	((si)->evtchn_pending[ix] & ~(si)->evtchn_mask[ix] & \
		(cpe)->evt_affinity[ix])

/*
 * This is the entry point for processing events from xen
 *
 * (See the commentary associated with the shared_info_st structure
 * in hypervisor-if.h)
 *
 * Since the event channel mechanism doesn't really implement the
 * concept of priority like hardware interrupt controllers, we simulate
 * that in software here using the cpu priority field and the pending
 * interrupts field.  Events/interrupts that are not able to be serviced
 * now because they are at a lower priority than the current cpu priority
 * cause a level bit to be recorded in the pending interrupts word.  When
 * the priority is lowered (either by spl or interrupt exit code) the pending
 * levels are checked and an upcall is scheduled if there are events/interrupts
 * that have become deliverable.
 */
void
xen_callback_handler(struct regs *rp, trap_trace_rec_t *ttp)
{
	ulong_t pending_sels, pe, selbit;
	int i, j, port, pri, curpri, irq;
	uint16_t pending_ints;
	struct cpu *cpu = CPU;
	volatile shared_info_t *si = HYPERVISOR_shared_info;
	volatile vcpu_info_t *vci = cpu->cpu_m.mcpu_vcpu_info;
	volatile struct xen_evt_data *cpe = cpu->cpu_m.mcpu_evt_pend;
	volatile uint16_t *cpu_ipp = &cpu->cpu_m.mcpu_intr_pending;

	ASSERT(rp->r_trapno == T_AST && rp->r_err == 0);
	ASSERT(&si->vcpu_info[cpu->cpu_id] == vci);
	ASSERT_STACK_ALIGNED();

	vci->evtchn_upcall_pending = 0;

	/*
	 * To expedite scanning of pending notifications, any 0->1
	 * pending transition on an unmasked channel causes a
	 * corresponding bit in evtchn_pending_sel to be set.
	 * Each bit in the selector covers a 32-bit word in
	 * the evtchn_pending[] array.
	 */
	membar_enter();
	do {
		pending_sels = vci->evtchn_pending_sel;
	} while (atomic_cas_ulong((volatile ulong_t *)&vci->evtchn_pending_sel,
	    pending_sels, 0) != pending_sels);

	pending_ints = *cpu_ipp;
	while ((i = ffs(pending_sels)) != 0) {
		i--;
		selbit = 1ul << i;
		pending_sels &= ~selbit;

		membar_enter();
		while ((pe = UNBLOCKED_EVENTS(si, i, cpe, cpu->cpu_id)) != 0) {
			j = ffs(pe) - 1;
			pe &= ~(1ul << j);

			port = (i << EVTCHN_SHIFT) + j;

			irq = evtchn_to_irq[port];

			/*
			 * If no irq set, just ignore the event.
			 * On e.g. netbsd they call evtchn_device_upcall(port)
			 * We require the evtchn driver to install a handler
			 * so there will be an irq associated with user mode
			 * evtchns.
			 */
			if (irq == INVALID_IRQ) {
				ec_clear_evtchn(port);
				continue;
			}

			/*
			 * If there's no handler, it could be a poke, so just
			 * accept the event and continue.
			 */
			if (!irq_info[irq].ii_u2.has_handler) {
#ifdef TRAPTRACE
				ttp->ttr_ipl = 0xff;
				if (IRQ_IS_CPUPOKE(irq)) {
					ttp->ttr_ipl = XC_CPUPOKE_PIL;
					ttp->ttr_marker = TT_INTERRUPT;
				}
				ttp->ttr_pri = cpu->cpu_pri;
				ttp->ttr_spl = cpu->cpu_base_spl;
				ttp->ttr_vector = 0xff;
#endif /* TRAPTRACE */
				if (ec_mask_evtchn(port)) {
					ec_clear_evtchn(port);
					ec_unmask_evtchn(port);
					continue;
				}
			}

			pri = irq_info[irq].ii_u2.ipl;

			/*
			 * If we are the cpu that successfully masks
			 * the event, then record it as a pending event
			 * for this cpu to service
			 */
			if (ec_mask_evtchn(port)) {
				if (ec_evtchn_pending(port)) {
					cpe->pending_sel[pri] |= selbit;
					cpe->pending_evts[pri][i] |= (1ul << j);
					pending_ints |= 1 << pri;
				} else {
					/*
					 * another cpu serviced this event
					 * before us, clear the mask.
					 */
					ec_unmask_evtchn(port);
				}
			}
		}
	}
	*cpu_ipp = pending_ints;
	if (pending_ints == 0)
		return;
	/*
	 * We have gathered all the pending events/interrupts,
	 * go service all the ones we can from highest priority to lowest.
	 * Note: This loop may not actually complete and service all
	 * pending interrupts since one of the interrupt threads may
	 * block and the pinned thread runs.  In that case, when we
	 * exit the interrupt thread that blocked we will check for
	 * any unserviced interrupts and re-post an upcall to process
	 * any unserviced pending events.
	 */
	curpri = cpu->cpu_pri;
	for (pri = bsrw_insn(*cpu_ipp); pri > curpri; pri--) {
		while ((pending_sels = cpe->pending_sel[pri]) != 0) {
			i = ffs(pending_sels) - 1;
			while ((pe = cpe->pending_evts[pri][i]) != 0) {
				j = ffs(pe) - 1;
				pe &= ~(1ul << j);
				cpe->pending_evts[pri][i] = pe;
				if (pe == 0) {
					/*
					 * Must reload pending selector bits
					 * here as they could have changed on
					 * a previous trip around the inner loop
					 * while we were interrupt enabled
					 * in a interrupt service routine.
					 */
					pending_sels = cpe->pending_sel[pri];
					pending_sels &= ~(1ul << i);
					cpe->pending_sel[pri] = pending_sels;
					if (pending_sels == 0)
						*cpu_ipp &= ~(1 << pri);
				}
				port = (i << EVTCHN_SHIFT) + j;
				irq = evtchn_to_irq[port];
				if (irq == INVALID_IRQ) {
					/*
					 * No longer a handler for this event
					 * channel.  Clear the event and
					 * ignore it, unmask the event.
					 */
					ec_clear_evtchn(port);
					ec_unmask_evtchn(port);
					continue;
				}
				if (irq == ec_dev_irq) {
					volatile int *tptr = &ec_dev_mbox;

					ASSERT(ec_dev_mbox == 0);
					/*
					 * NOTE: this gross store thru a pointer
					 * is necessary because of a Sun C
					 * compiler bug that does not properly
					 * honor a volatile declaration.
					 * we really should just be able to say
					 * 	ec_dev_mbox = port;
					 * here
					 */
					*tptr = port;
				}
				/*
				 * Set up the regs struct to
				 * look like a normal hardware int
				 * and do normal interrupt handling.
				 */
				rp->r_trapno = irq;
				do_interrupt(rp, ttp);
				/*
				 * Check for cpu priority change
				 * Can happen if int thread blocks
				 */
				if (cpu->cpu_pri > curpri)
					return;
			}
		}
	}
}

void
ec_unmask_evtchn(unsigned int ev)
{
	uint_t evi;
	volatile shared_info_t *si = HYPERVISOR_shared_info;
	volatile vcpu_info_t *vci = CPU->cpu_m.mcpu_vcpu_info;
	volatile ulong_t *ulp;

	ASSERT(!interrupts_enabled());
	/*
	 * Check if we need to take slow path
	 */
	if (!CPU_IN_SET(evtchn_cpus[ev], CPU->cpu_id)) {
		xen_evtchn_unmask(ev);
		return;
	}
	evi = ev >> EVTCHN_SHIFT;
	ev &= (1ul << EVTCHN_SHIFT) - 1;
	ulp = (volatile ulong_t *)&si->evtchn_mask[evi];
	atomic_and_ulong(ulp, ~(1ul << ev));
	/*
	 * The following is basically the equivalent of
	 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose the
	 * interrupt edge' if the channel is masked.
	 * XXPV - slight race if upcall was about to be set, we may get
	 * an extra upcall.
	 */
	membar_enter();
	if (si->evtchn_pending[evi] & (1ul << ev)) {
		membar_consumer();
		ulp = (volatile ulong_t *)&vci->evtchn_pending_sel;
		if (!(*ulp & (1ul << evi))) {
			atomic_or_ulong(ulp, (1ul << evi));
		}
		vci->evtchn_upcall_pending = 1;
	}
}

/*
 * Set a bit in an evtchan mask word, return true if we are the cpu that
 * set the bit.
 */
int
ec_mask_evtchn(unsigned int ev)
{
	uint_t evi, evb;
	ulong_t new, old, bit;
	volatile shared_info_t *si = HYPERVISOR_shared_info;
	volatile ulong_t *maskp;
	int masked;

	kpreempt_disable();
	evi = ev >> EVTCHN_SHIFT;
	evb = ev & ((1ul << EVTCHN_SHIFT) - 1);
	bit = 1ul << evb;
	maskp = (volatile ulong_t *)&si->evtchn_mask[evi];
	do {
		old = si->evtchn_mask[evi];
		new = old | bit;
	} while (atomic_cas_ulong(maskp, old, new) != old);
	masked = (old & bit) == 0;
	if (masked) {
		evtchn_owner[ev] = CPU->cpu_id;
#ifdef DEBUG
		evtchn_owner_thread[ev] = curthread;
#endif
	}
	kpreempt_enable();
	return (masked);
}

void
ec_clear_evtchn(unsigned int ev)
{
	uint_t evi;
	shared_info_t *si = HYPERVISOR_shared_info;
	volatile ulong_t *pendp;

	evi = ev >> EVTCHN_SHIFT;
	ev &= (1ul << EVTCHN_SHIFT) - 1;
	pendp = (volatile ulong_t *)&si->evtchn_pending[evi];
	atomic_and_ulong(pendp, ~(1ul << ev));
}

void
ec_notify_via_evtchn(unsigned int port)
{
	evtchn_send_t send;

	ASSERT(port != INVALID_EVTCHN);

	send.port = port;
	(void) HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
}

int
ec_block_irq(int irq)
{
	irq_info_t *irqp = &irq_info[irq];
	int evtchn;


	evtchn = irq_evtchn(irqp);
	(void) ec_mask_evtchn(evtchn);
	return (evtchn_owner[evtchn]);
}

/*
 * Make a event that is pending for delivery on the current cpu  "go away"
 * without servicing the interrupt.
 */
void
ec_unpend_irq(int irq)
{
	irq_info_t *irqp = &irq_info[irq];
	int pri = irqp->ii_u2.ipl;
	ulong_t flags;
	uint_t evtchn, evi, bit;
	unsigned long pe, pending_sels;
	struct xen_evt_data *cpe;

	/*
	 * The evtchn must be masked
	 */
	evtchn = irq_evtchn(irqp);
	ASSERT(EVTCHN_MASKED(evtchn));
	evi = evtchn >> EVTCHN_SHIFT;
	bit = evtchn & (1ul << EVTCHN_SHIFT) - 1;
	flags = intr_clear();
	cpe = CPU->cpu_m.mcpu_evt_pend;
	pe = cpe->pending_evts[pri][evi] & ~(1ul << bit);
	cpe->pending_evts[pri][evi] = pe;
	if (pe == 0) {
		pending_sels = cpe->pending_sel[pri];
		pending_sels &= ~(1ul << evi);
		cpe->pending_sel[pri] = pending_sels;
		if (pending_sels == 0)
			CPU->cpu_m.mcpu_intr_pending &= ~(1 << pri);
	}
	intr_restore(flags);
}