common/os/softint.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/types.h>
#include <sys/param.h>
#include <sys/t_lock.h>
#include <sys/systm.h>
#include <sys/spl.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/kdi_impl.h>
#include <sys/cpuvar.h>
#include <sys/cpuvar.h>
#include <sys/archsystm.h>

/*
 * Handle software interrupts through 'softcall' mechanism
 *
 * At present softcall mechanism uses a global list headed by softhead.
 * Entries are added to tail and removed from head so as to preserve FIFO
 * nature of entries in the softcall list. softcall() takes care of adding
 * entries to the softtail.
 *
 * softint must take care of executing the entries in the FIFO
 * order. It could be called simultaneously from multiple cpus, however only
 * one instance of softint should process the softcall list with the exception
 * when CPU is stuck due to high interrupt load and can't execute callbacks.
 * State diagram is as follows :-
 *
 *	- Upper half which is same as old state machine
 *	  (IDLE->PEND->DRAIN->IDLE)
 *
 *	- Lower half which steals the entries from softcall queue and execute
 *        in the context of softint interrupt handler. The interrupt handler
 *        is fired on a different CPU by sending a cross-call.
 *
 * Starting state is IDLE.
 *
 * 				softint()
 *
 *
 *				(c)
 * 	____________________________________________________
 * 	|                          ^                         ^
 * 	v            (a)           |           (b)           |
 * 	IDLE--------------------->PEND--------------------->DRAIN
 *	^                         |                         |
 * 	|                         |                         |
 * 	|                         |                         |
 * 	|                         |                         |
 * 	|                         |                         |
 * 	|                         d                         d
 * 	|                         |                         |
 * 	|                         v                         v
 * 	|                         PEND                      DRAIN
 * 	|            (e)           &                          &
 * 	|<-----------------------STEAL                      STEAL
 * 	^                                                    |
 * 	|                                                    |
 * 	|                         (e)                        v
 * 	|_________________________<__________________________|
 *
 *
 *
 * Edge (a)->(b)->(c) are same as old state machine and these
 * are mutually exclusive state.
 *
 * a - When an entry is being enqueued to softcall queue then the state
 *     moves from IDLE to PEND.
 *
 * b - When interrupt handler has started processing softcall queue.
 *
 * c - When interrupt handler finished processing softcall queue, the
 *     state of machines goes back to IDLE.
 *
 * d - softcall() generates another softlevel1 iff interrupt handler
 *     hasn't run recently.
 *
 * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1
 *     handler exit because we have processed all the entries.
 *
 * When CPU is being pinned by higher level interrupts for more than
 * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1
 * handler on the other CPU can drain the queue.
 *
 * These states are needed for softcall mechanism since Solaris has only
 * one interface (ie. siron ) as of now for :
 *
 * - raising a soft interrupt architecture independently (ie not through
 *   setsoftint(..) )
 * - to process the softcall queue.
 */

#define	NSOFTCALLS	200

/*
 * Defined states for softcall processing.
 */
#define	SOFT_IDLE		0x01	/* no processing is needed */
#define	SOFT_PEND		0x02	/* softcall list needs processing */
#define	SOFT_DRAIN		0x04	/* list is being processed */
#define	SOFT_STEAL		0x08	/* list is being stolen for draining */

typedef struct softcall {
	void (*sc_func)(void *);	/* function to call */
	void *sc_arg;			/* arg to pass to func */
	struct softcall *sc_next;	/* next in list */
} softcall_t;

/*
 * softcall list and state variables.
 */
static softcall_t *softcalls;
static softcall_t *softhead, *softtail, *softfree;
static uint_t	softcall_state;
static clock_t softcall_tick;
static clock_t softcall_countstart, softcall_lastpoke;
static uint_t softcall_pokecount;

/*
 * Max number of pokes per second before increasing softcall_delay
 */
uint_t softcall_pokemax = 10;

/*
 * This ensures that softcall entries don't get stuck for long. It's expressed
 * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency
 * is used, softcall_init() ensures that it's still expressed as 1 =  10 milli
 * seconds.
 */
unsigned int softcall_delay = 1;

/*
 * The last CPU which will drain softcall queue.
 */
static int softcall_latest_cpuid = -1;

/*
 * CPUSET to hold the CPU which is processing softcall queue
 * currently. There can be more than one CPU having bit set
 * but it will happen only when they are stuck.
 */
static cpuset_t *softcall_cpuset = NULL;

/*
 * protects softcall lists and control variable softcall_state.
 */
static kmutex_t	softcall_lock;

static void (*kdi_softcall_func)(void);
extern void siron_poke_cpu(cpuset_t);

extern void siron(void);
extern void kdi_siron(void);


void
softcall_init(void)
{
	softcall_t *sc;

	softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP);
	softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP);
	for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) {
		sc->sc_next = softfree;
		softfree = sc;
	}
	mutex_init(&softcall_lock, NULL, MUTEX_SPIN,
	    (void *)ipltospl(SPL8));
	softcall_state = SOFT_IDLE;
	softcall_tick = lbolt;

	/*
	 * Since softcall_delay is expressed as 1 = 10 milliseconds.
	 */
	softcall_delay = softcall_delay * (hz/100);
	CPUSET_ZERO(*softcall_cpuset);
}

/*
 * Gets called when softcall queue is not moving forward. We choose
 * a CPU and poke except the ones which are already poked.
 */
static int
softcall_choose_cpu()
{
	cpu_t *cplist = CPU;
	cpu_t *cp;
	int intr_load = INT_MAX;
	int cpuid = -1;
	cpuset_t poke;
	int s;

	ASSERT(getpil() >= DISP_LEVEL);
	ASSERT(ncpus > 1);
	ASSERT(MUTEX_HELD(&softcall_lock));

	CPUSET_ZERO(poke);

	/*
	 * The hint is to start from current CPU.
	 */
	cp = cplist;
	do {
		/*
		 * Don't select this CPU if :
		 *   - in cpuset already
		 *   - CPU is not accepting interrupts
		 *   - CPU is being offlined
		 */
		if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) ||
		    (cp->cpu_flags & CPU_ENABLE) == 0 ||
		    (cp == cpu_inmotion))
			continue;
#if defined(__x86)
		/*
		 * Don't select this CPU if a hypervisor indicates it
		 * isn't currently scheduled onto a physical cpu.  We are
		 * looking for a cpu that can respond quickly and the time
		 * to get the virtual cpu scheduled and switched to running
		 * state is likely to be relatively lengthy.
		 */
		if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU)
			continue;
#endif	/* __x86 */

		/* if CPU is not busy */
		if (cp->cpu_intrload == 0) {
			cpuid = cp->cpu_id;
			break;
		}

		if (cp->cpu_intrload < intr_load) {
			cpuid = cp->cpu_id;
			intr_load = cp->cpu_intrload;
		} else if (cp->cpu_intrload == intr_load) {
			/*
			 * We want to poke CPUs having similar
			 * load because we don't know which CPU is
			 * can acknowledge level1 interrupt. The
			 * list of such CPUs should not be large.
			 */
			if (cpuid != -1) {
				/*
				 * Put the last CPU chosen because
				 * it also has same interrupt load.
				 */
				CPUSET_ADD(poke, cpuid);
				cpuid = -1;
			}

			CPUSET_ADD(poke, cp->cpu_id);
		}
	} while ((cp = cp->cpu_next_onln) != cplist);

	/* if we found a CPU which suits best to poke */
	if (cpuid != -1) {
		CPUSET_ZERO(poke);
		CPUSET_ADD(poke, cpuid);
	}

	if (CPUSET_ISNULL(poke)) {
		mutex_exit(&softcall_lock);
		return (0);
	}

	/*
	 * We first set the bit in cpuset and then poke.
	 */
	CPUSET_XOR(*softcall_cpuset, poke);
	mutex_exit(&softcall_lock);

	/*
	 * If softcall() was called at low pil then we may
	 * get preempted before we raise PIL. It should be okay
	 * because we are just going to poke CPUs now or at most
	 * another thread may start choosing CPUs in this routine.
	 */
	s = splhigh();
	siron_poke_cpu(poke);
	splx(s);
	return (1);
}


/*
 * Call function func with argument arg
 * at some later time at software interrupt priority
 */
void
softcall(void (*func)(void *), void *arg)
{
	softcall_t *sc;
	clock_t w, now;

	/*
	 * protect against cross-calls
	 */
	mutex_enter(&softcall_lock);
	/* coalesce identical softcalls */
	for (sc = softhead; sc != 0; sc = sc->sc_next) {
		if (sc->sc_func == func && sc->sc_arg == arg) {
			goto intr;
		}
	}

	if ((sc = softfree) == 0)
		panic("too many softcalls");

	softfree = sc->sc_next;
	sc->sc_func = func;
	sc->sc_arg = arg;
	sc->sc_next = 0;

	if (softhead) {
		softtail->sc_next = sc;
		softtail = sc;
	} else
		softhead = softtail = sc;

intr:
	if (softcall_state & SOFT_IDLE) {
		softcall_state = SOFT_PEND;
		softcall_tick = lbolt;
		mutex_exit(&softcall_lock);
		siron();
	} else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) {
		now = lbolt;
		w = now - softcall_tick;
		if (w <= softcall_delay || ncpus == 1) {
			mutex_exit(&softcall_lock);
			return;
		}
		/*
		 * Did we poke less than a second ago?
		 */
		if (now - softcall_lastpoke < hz) {
			/*
			 * We did, increment the poke count and
			 * see if we are poking too often
			 */
			if (softcall_pokecount++ == 0)
				softcall_countstart = now;
			if (softcall_pokecount > softcall_pokemax) {
				/*
				 * If poking too much increase the delay
				 */
				if (now - softcall_countstart <= hz)
					softcall_delay++;
				softcall_pokecount = 0;
			}
		} else {
			/*
			 * poke rate has dropped off, reset the poke monitor
			 */
			softcall_pokecount = 0;
		}
		softcall_lastpoke = lbolt;
		if (!(softcall_state & SOFT_STEAL)) {
			softcall_state |= SOFT_STEAL;

			/*
			 * We want to give some more chance before
			 * fishing around again.
			 */
			softcall_tick = lbolt;
		}

		/* softcall_lock will be released by this routine */
		(void) softcall_choose_cpu();
	}
}

void
kdi_softcall(void (*func)(void))
{
	kdi_softcall_func = func;

	if (softhead == NULL)
		kdi_siron();
}

/*
 * Called to process software interrupts take one off queue, call it,
 * repeat.
 *
 * Note queue may change during call; softcall_lock, state variables
 * softcall_state and softcall_latest_cpuid ensures that -
 * - we don't have multiple cpus pulling from the list (thus causing
 *   a violation of FIFO order with an exception when we are stuck).
 * - we don't miss a new entry having been added to the head.
 * - we don't miss a wakeup.
 */

void
softint(void)
{
	softcall_t *sc = NULL;
	void (*func)();
	caddr_t arg;
	int cpu_id = CPU->cpu_id;

	/*
	 * Don't process softcall queue if current CPU is quiesced or
	 * offlined. This can happen when a CPU is running pause
	 * thread but softcall already sent a xcall.
	 */
	if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) {
		if (softcall_cpuset != NULL &&
		    CPU_IN_SET(*softcall_cpuset, cpu_id)) {
			CPUSET_DEL(*softcall_cpuset, cpu_id);
			goto out;
		}
	}

	mutex_enter(&softcall_lock);

	if (softcall_state & (SOFT_STEAL|SOFT_PEND)) {
		softcall_state = SOFT_DRAIN;
	} else  {
		/*
		 * The check for softcall_cpuset being
		 * NULL is required because it may get
		 * called very early during boot.
		 */
		if (softcall_cpuset != NULL &&
		    CPU_IN_SET(*softcall_cpuset, cpu_id))
			CPUSET_DEL(*softcall_cpuset, cpu_id);
		mutex_exit(&softcall_lock);
		goto out;
	}

	/*
	 * Setting softcall_latest_cpuid to current CPU ensures
	 * that there is only one active softlevel1 handler to
	 * process softcall queues.
	 *
	 * Since softcall_lock lock is dropped before calling
	 * func (callback), we need softcall_latest_cpuid
	 * to prevent two softlevel1 hanlders working on the
	 * queue when the first softlevel1 handler gets
	 * stuck due to high interrupt load.
	 */
	softcall_latest_cpuid = cpu_id;

	/* add ourself to the cpuset */
	if (!CPU_IN_SET(*softcall_cpuset, cpu_id))
		CPUSET_ADD(*softcall_cpuset, cpu_id);

	for (;;) {
		softcall_tick = lbolt;
		if ((sc = softhead) != NULL) {
			func = sc->sc_func;
			arg = sc->sc_arg;
			softhead = sc->sc_next;
			sc->sc_next = softfree;
			softfree = sc;
		}

		if (sc == NULL) {
			if (CPU_IN_SET(*softcall_cpuset, cpu_id))
				CPUSET_DEL(*softcall_cpuset, cpu_id);

			softcall_state = SOFT_IDLE;
			ASSERT(softcall_latest_cpuid == cpu_id);
			softcall_latest_cpuid = -1;

			mutex_exit(&softcall_lock);
			break;
		}

		mutex_exit(&softcall_lock);
		func(arg);
		mutex_enter(&softcall_lock);

		/*
		 * No longer need softcall processing from current
		 * interrupt handler because either
		 *  (a) softcall is in SOFT_IDLE state or
		 *  (b) There is a CPU already draining softcall
		 *	queue and the current softlevel1 is no
		 *	longer required.
		 */
		if (softcall_latest_cpuid != cpu_id) {
			if (CPU_IN_SET(*softcall_cpuset, cpu_id))
				CPUSET_DEL(*softcall_cpuset, cpu_id);

			mutex_exit(&softcall_lock);
			break;
		}
	}

out:
	if ((func = kdi_softcall_func) != NULL) {
		kdi_softcall_func = NULL;
		func();
	}
}