audio/impl/audio_engine.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 4Front Technologies 1996-2008.
 *
 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/types.h>
#include <sys/list.h>
#include <sys/sysmacros.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/callb.h>
#include <sys/kstat.h>
#include <sys/note.h>

#include "audio_impl.h"

/*
 * Audio Engine functions.
 */

/*
 * Globals
 */
uint_t		audio_intrhz = AUDIO_INTRHZ;
/*
 * We need to operate at fairly high interrupt priority to avoid
 * underruns due to other less time sensitive processing.
 */
int		audio_priority = DDI_IPL_8;

audio_dev_t *
audio_dev_alloc(dev_info_t *dip, int instance)
{
	audio_dev_t *d;

	/*
	 * For a card with multiple independent audio ports on it, we
	 * allow the driver to provide a different instance numbering
	 * scheme than the standard DDI instance number.  (This is
	 * sort of like the PPA numbering scheme used by NIC drivers
	 * -- by default PPA == instance, but sometimes we need more
	 * flexibility.)
	 */
	if (instance == 0) {
		instance = ddi_get_instance(dip);
	}
	/* generally this shouldn't occur */
	if (instance > AUDIO_MN_INST_MASK) {
		audio_dev_warn(NULL, "bad instance number for %s (%d)",
		    ddi_driver_name(dip), instance);
		return (NULL);
	}

	if ((d = kmem_zalloc(sizeof (*d), KM_NOSLEEP)) == NULL) {
		audio_dev_warn(NULL, "unable to allocate audio device struct");
		return (NULL);
	}
	d->d_dip = dip;
	d->d_number = -1;
	d->d_major = ddi_driver_major(dip);
	d->d_instance = instance;
	d->d_pcmvol = 100;
	mutex_init(&d->d_lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&d->d_cv, NULL, CV_DRIVER, NULL);
	mutex_init(&d->d_ctrl_lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&d->d_ctrl_cv, NULL, CV_DRIVER, NULL);
	list_create(&d->d_clients, sizeof (struct audio_client),
	    offsetof(struct audio_client, c_dev_linkage));
	list_create(&d->d_engines, sizeof (struct audio_engine),
	    offsetof(struct audio_engine, e_dev_linkage));
	list_create(&d->d_controls, sizeof (struct audio_ctrl),
	    offsetof(struct audio_ctrl, ctrl_linkage));
	list_create(&d->d_hwinfo, sizeof (struct audio_infostr),
	    offsetof(struct audio_infostr, i_linkage));
	(void) snprintf(d->d_name, sizeof (d->d_name), "%s#%d",
	    ddi_driver_name(dip), instance);

	return (d);
}

void
audio_dev_free(audio_dev_t *d)
{
	struct audio_infostr *isp;

	while ((isp = list_remove_head(&d->d_hwinfo)) != NULL) {
		kmem_free(isp, sizeof (*isp));
	}
	if (d->d_pcmvol_ctrl != NULL) {
		audio_dev_del_control(d->d_pcmvol_ctrl);
	}
	list_destroy(&d->d_hwinfo);
	list_destroy(&d->d_engines);
	list_destroy(&d->d_controls);
	list_destroy(&d->d_clients);
	mutex_destroy(&d->d_ctrl_lock);
	mutex_destroy(&d->d_lock);
	cv_destroy(&d->d_cv);
	cv_destroy(&d->d_ctrl_cv);
	kmem_free(d, sizeof (*d));
}

void
audio_dev_set_description(audio_dev_t *d, const char *desc)
{
	(void) strlcpy(d->d_desc, desc, sizeof (d->d_desc));
}

void
audio_dev_set_version(audio_dev_t *d, const char *vers)
{
	(void) strlcpy(d->d_vers, vers, sizeof (d->d_vers));
}

void
audio_dev_add_info(audio_dev_t *d, const char *info)
{
	struct audio_infostr *isp;

	/* failure to add information structure is not critical */
	isp = kmem_zalloc(sizeof (*isp), KM_NOSLEEP);
	if (isp == NULL) {
		audio_dev_warn(d, "unable to allocate information structure");
	} else {
		(void) snprintf(isp->i_line, sizeof (isp->i_line), info);
		list_insert_tail(&d->d_hwinfo, isp);
	}
}

static void
auimpl_engine_reset(audio_engine_t *e)
{
	char	*buf;
	char	*ptr;
	int	nfr, resid, cnt;
	int	tidx;

	tidx = e->e_tidx;
	nfr = min(e->e_head - e->e_tail, e->e_nframes);
	buf = kmem_alloc(nfr * e->e_framesz, KM_SLEEP);
	ptr = buf;
	cnt = 0;

	ASSERT(e->e_nframes);

	for (resid = nfr; resid; resid -= cnt) {
		int	nbytes;

		cnt = min((e->e_nframes - tidx), resid);
		nbytes = cnt * e->e_framesz;

		bcopy(e->e_data + (tidx * e->e_framesz), ptr, nbytes);
		ptr += nbytes;
		tidx += cnt;
		if (tidx == e->e_nframes) {
			tidx = 0;
		}
	}

	if (e->e_flags & ENGINE_INPUT) {
		/* record */
		e->e_hidx = 0;
		e->e_tidx = (e->e_nframes - nfr) % e->e_nframes;
	} else {
		/* play */
		e->e_hidx = nfr % e->e_nframes;
		e->e_tidx = 0;
	}

	/* relocate from scratch area to destination */
	bcopy(buf, e->e_data + (e->e_tidx * e->e_framesz), nfr * e->e_framesz);
	kmem_free(buf, nfr * e->e_framesz);
}

static volatile uint_t auimpl_engno = 0;

audio_engine_t *
audio_engine_alloc(audio_engine_ops_t *ops, uint_t flags)
{
	int i;
	audio_engine_t *e;
	char tname[32];
	int num;

	if (ops->audio_engine_version != AUDIO_ENGINE_VERSION) {
		audio_dev_warn(NULL, "audio engine version mismatch: %d != %d",
		    ops->audio_engine_version, AUDIO_ENGINE_VERSION);
		return (NULL);
	}

	/* NB: The ops vector must be held in persistent storage! */
	e = kmem_zalloc(sizeof (audio_engine_t), KM_NOSLEEP);
	if (e == NULL) {
		audio_dev_warn(NULL, "unable to allocate engine struct");
		return (NULL);
	}
	e->e_ops = *ops;
	mutex_init(&e->e_lock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(audio_priority));
	cv_init(&e->e_cv, NULL, CV_DRIVER, NULL);
	list_create(&e->e_streams, sizeof (struct audio_stream),
	    offsetof(struct audio_stream, s_eng_linkage));

	for (i = 0; i < AUDIO_MAX_CHANNELS; i++) {
		e->e_chbufs[i] = kmem_zalloc(sizeof (int32_t) * AUDIO_CHBUFS,
		    KM_NOSLEEP);
		if (e->e_chbufs[i] == NULL) {
			audio_dev_warn(NULL, "unable to allocate channel buf");
			audio_engine_free(e);
			return (NULL);
		}
	}

	num = atomic_inc_uint_nv(&auimpl_engno);

	(void) snprintf(tname, sizeof (tname), "audio_engine_%d", num);

	e->e_flags = flags & ENGINE_DRIVER_FLAGS;
	return (e);
}

void
audio_engine_free(audio_engine_t *e)
{
	int i;

	for (i = 0; i < AUDIO_MAX_CHANNELS; i++) {
		if (e->e_chbufs[i] != NULL) {
			kmem_free(e->e_chbufs[i],
			    sizeof (int32_t) * AUDIO_CHBUFS);
		}
	}

	list_destroy(&e->e_streams);
	mutex_destroy(&e->e_lock);
	cv_destroy(&e->e_cv);
	kmem_free(e, sizeof (*e));
}

static list_t auimpl_devs_by_index;
static list_t auimpl_devs_by_number;
static krwlock_t auimpl_dev_lock;

/*
 * Not for public consumption: Private interfaces.
 */
void
auimpl_dev_hold(audio_dev_t *d)
{
	/* bump the reference count */
	mutex_enter(&d->d_lock);
	d->d_refcnt++;
	mutex_exit(&d->d_lock);
}

audio_dev_t *
auimpl_dev_hold_by_devt(dev_t dev)
{
	audio_dev_t *d;
	major_t major;
	int instance;
	list_t *l = &auimpl_devs_by_index;

	major = getmajor(dev);
	instance = (getminor(dev) >> AUDIO_MN_INST_SHIFT) & AUDIO_MN_INST_MASK;

	rw_enter(&auimpl_dev_lock, RW_READER);

	for (d = list_head(l); d; d = list_next(l, d)) {
		if ((d->d_major == major) && (d->d_instance == instance)) {
			auimpl_dev_hold(d);
			break;
		}
	}

	rw_exit(&auimpl_dev_lock);
	return (d);
}

audio_dev_t *
auimpl_dev_hold_by_index(int index)
{
	audio_dev_t *d;
	list_t *l = &auimpl_devs_by_index;

	rw_enter(&auimpl_dev_lock, RW_READER);

	for (d = list_head(l); d; d = list_next(l, d)) {
		if (d->d_index == index) {
			auimpl_dev_hold(d);
			break;
		}
	}

	rw_exit(&auimpl_dev_lock);
	return (d);
}

void
auimpl_dev_release(audio_dev_t *d)
{
	mutex_enter(&d->d_lock);
	d->d_refcnt--;
	mutex_exit(&d->d_lock);
}

int
auimpl_choose_format(int fmts)
{
	/*
	 * Choose the very best format we can.  We choose 24 bit in
	 * preference to 32 bit because we mix in 24 bit.  We do that
	 * to allow overflows to fit within 32-bits.  (Very few humans
	 * can tell a difference between 24 and 32 bit audio anyway.)
	 */
	if (fmts & AUDIO_FORMAT_S24_NE)
		return (AUDIO_FORMAT_S24_NE);

	if (fmts & AUDIO_FORMAT_S32_NE)
		return (AUDIO_FORMAT_S32_NE);

	if (fmts & AUDIO_FORMAT_S24_OE)
		return (AUDIO_FORMAT_S24_OE);

	if (fmts & AUDIO_FORMAT_S32_OE)
		return (AUDIO_FORMAT_S32_OE);

	if (fmts & AUDIO_FORMAT_S16_NE)
		return (AUDIO_FORMAT_S16_NE);

	if (fmts & AUDIO_FORMAT_S16_OE)
		return (AUDIO_FORMAT_S16_OE);

	if (fmts & AUDIO_FORMAT_AC3)
		return (AUDIO_FORMAT_AC3);

	return (AUDIO_FORMAT_NONE);
}

int
auimpl_engine_open(audio_stream_t *sp, int flags)
{
	return (auimpl_engine_setup(sp, flags, NULL, FORMAT_MSK_NONE));
}


int
auimpl_engine_setup(audio_stream_t *sp, int flags, audio_parms_t *parms,
    uint_t mask)
{
	audio_dev_t	*d = sp->s_client->c_dev;
	audio_engine_t	*e = NULL;
	audio_parms_t	uparms;
	list_t		*list;
	uint_t		cap;
	int		priority = 0;
	int		rv = ENODEV;
	int		sampsz;
	int		i;
	int		fragfr;
	int		fmts;


	mutex_enter(&d->d_lock);

	uparms = *sp->s_user_parms;
	if (mask & FORMAT_MSK_FMT)
		uparms.p_format = parms->p_format;
	if (mask & FORMAT_MSK_RATE)
		uparms.p_rate = parms->p_rate;
	if (mask & FORMAT_MSK_CHAN)
		uparms.p_nchan = parms->p_nchan;

	/*
	 * Which direction are we opening?  (We must open exactly
	 * one direction, otherwise the open is meaningless.)
	 */

	if (sp == &sp->s_client->c_ostream) {
		cap = ENGINE_OUTPUT_CAP;
		flags |= ENGINE_OUTPUT;
	} else {
		cap = ENGINE_INPUT_CAP;
		flags |= ENGINE_INPUT;
	}

	if (uparms.p_format == AUDIO_FORMAT_AC3) {
		fmts = AUDIO_FORMAT_AC3;
		flags |= ENGINE_EXCLUSIVE;
	} else {
		fmts = AUDIO_FORMAT_PCM;
	}

	list = &d->d_engines;


	/* If the device is suspended, wait for it to resume. */
	while (d->d_suspended) {
		cv_wait(&d->d_ctrl_cv, &d->d_lock);
	}

again:

	for (audio_engine_t *t = list_head(list); t; t = list_next(list, t)) {
		int		mypri;
		int		r;

		/* Make sure the engine can do what we want it to. */
		mutex_enter(&t->e_lock);

		if ((t->e_flags & cap) == 0) {
			mutex_exit(&t->e_lock);
			continue;
		}

		/*
		 * Open the engine early, as the inquiries to rate and format
		 * may not be accurate until this is done.
		 */
		if (list_is_empty(&t->e_streams)) {
			if (ENG_OPEN(t, flags, &t->e_nframes, &t->e_data)) {
				mutex_exit(&t->e_lock);
				rv = EIO;
				continue;
			}
		}

		if ((ENG_FORMAT(t) & fmts) == 0) {
			if (list_is_empty(&t->e_streams))
				ENG_CLOSE(t);
			mutex_exit(&t->e_lock);
			continue;
		}


		/* If it is in failed state, don't use this engine. */
		if (t->e_failed) {
			if (list_is_empty(&t->e_streams))
				ENG_CLOSE(t);
			mutex_exit(&t->e_lock);
			rv = rv ? EIO : 0;
			continue;
		}

		/*
		 * If the engine is in exclusive use, we can't use it.
		 * This is intended for use with AC3 or digital
		 * streams that cannot tolerate mixing.
		 */
		if ((t->e_flags & ENGINE_EXCLUSIVE) && (t != sp->s_engine)) {
			if (list_is_empty(&t->e_streams))
				ENG_CLOSE(t);
			mutex_exit(&t->e_lock);
			rv = rv ? EBUSY : 0;
			continue;
		}

		/*
		 * If the engine is in use incompatibly, we can't use
		 * it.  This should only happen for half-duplex audio
		 * devices.  I've not seen any of these that are
		 * recent enough to be supported by Solaris.
		 */
		if (((flags & ENGINE_INPUT) && (t->e_flags & ENGINE_OUTPUT)) ||
		    ((flags & ENGINE_OUTPUT) && (t->e_flags & ENGINE_INPUT))) {
			if (list_is_empty(&t->e_streams))
				ENG_CLOSE(t);
			mutex_exit(&t->e_lock);
			/* Only override the ENODEV or EIO. */
			rv = rv ? EBUSY : 0;
			continue;
		}

		/*
		 * In order to support as many different possible
		 * output streams (e.g. AC3 passthru or AC3 decode),
		 * or multiple exclusive outputs, we treat audio
		 * engines as *precious*.
		 *
		 * This means that we will try hard to reuse an
		 * existing allocated engine.  This may not be the
		 * optimal performance configuration (especially if we
		 * wanted to avoid rate conversion, for example), but
		 * it should have fewer cases where the configuration
		 * results in denying service to any client.
		 */

		/*
		 * This engine *can* support us, so we should no longer
		 * have a failure mode.
		 */
		rv = 0;
		mypri = (1U << 0);


		/*
		 * Mixing is cheap, so try not to pick on idle
		 * engines.  This avoids burning bus bandwidth (which
		 * may be precious for certain classes of traffic).
		 * Note that idleness is given a low priority compared
		 * to the other considerations.
		 *
		 * We also use this opportunity open the engine, if
		 * not already done so, so that our parameter
		 * inquiries will be valid.
		 */
		if (!list_is_empty(&t->e_streams))
			mypri |= (1U << 1);

		/*
		 * Slight preference is given to reuse an engine that
		 * we might already be using.
		 */
		if (t == sp->s_engine)
			mypri |= (1U << 2);


		/*
		 * Sample rate conversion avoidance.  Upsampling
		 * requires multiplications and is moderately
		 * expensive.  Downsampling requires division and is
		 * quite expensive, and hence to be avoided if at all
		 * possible.
		 */
		r = ENG_RATE(t);
		if (uparms.p_rate == r) {
			/*
			 * No conversion needed at all.  This is ideal.
			 */
			mypri |= (1U << 4) | (1U << 3);
		} else {
			int src, dst;

			if (flags & ENGINE_INPUT) {
				src = r;
				dst = uparms.p_rate;
			} else {
				src = uparms.p_rate;
				dst = r;
			}
			if ((src < dst) && ((dst % src) == 0)) {
				/*
				 * Pure upsampling only. This
				 * penalizes any engine which requires
				 * downsampling.
				 */
				mypri |= (1U << 3);
			}
		}

		/*
		 * Try not to pick on duplex engines.  This way we
		 * leave engines that can be used for recording or
		 * playback available as such.  All modern drivers
		 * use separate unidirectional engines for playback
		 * and record.
		 */
		if ((t->e_flags & ENGINE_CAPS) == cap) {
			mypri |= (1U << 5);
		}

		/*
		 * Try not to pick on engines that can do other
		 * formats.  This will generally be false, but if it
		 * happens we pretty strongly avoid using a limited
		 * resource.
		 */
		if ((t->e_format & ~fmts) == 0) {
			mypri |= (1U << 6);
		}

		if (mypri > priority) {
			if (e != NULL) {
				/*
				 * If we opened this for our own use
				 * and we are no longer using it, then
				 * close it back down.
				 */
				if (list_is_empty(&e->e_streams))
					ENG_CLOSE(e);
				mutex_exit(&e->e_lock);
			}
			e = t;
			priority = mypri;
		} else {
			mutex_exit(&t->e_lock);
		}

		/*
		 * Locking: at this point, if we have an engine, "e", it is
		 * locked.  No other engines should have a lock held.
		 */
	}

	if ((rv == EBUSY) && ((flags & ENGINE_NDELAY) == 0)) {
		ASSERT(e == NULL);
		if (cv_wait_sig(&d->d_cv, &d->d_lock) == 0) {
			mutex_exit(&d->d_lock);
			return (EINTR);
		}
		goto again;
	}

	if (rv != 0) {
		ASSERT(e == NULL);
		mutex_exit(&d->d_lock);
		return (rv);
	}

	ASSERT(e != NULL);
	ASSERT(mutex_owned(&e->e_lock));

	if (sp->s_engine && (sp->s_engine != e)) {
		/*
		 * If this represents a potential engine change, then
		 * we close off everything, and start anew. This turns
		 * out to be vastly simpler than trying to close all
		 * the races associated with a true hand off.  This
		 * ought to be relatively uncommon (changing engines).
		 */

		/* Drop the new reference. */
		if (list_is_empty(&e->e_streams))
			ENG_CLOSE(e);
		mutex_exit(&e->e_lock);
		mutex_exit(&d->d_lock);

		auimpl_engine_close(sp);

		/* Try again. */
		return (auimpl_engine_setup(sp, flags, parms, mask));
	}

	if (sp->s_engine == NULL) {
		/*
		 * Add a reference to this engine if we don't already
		 * have one.
		 */
		sp->s_engine = e;

		if (!list_is_empty(&e->e_streams)) {
			/*
			 * If the engine is already open, there is no
			 * need for further work.  The first open will
			 * be relatively expensive, but subsequent
			 * opens should be as cheap as possible.
			 */
			list_insert_tail(&e->e_streams, sp);
			goto ok;
		}
		list_insert_tail(&e->e_streams, sp);

	} else {
		ASSERT(sp->s_engine == e);
		/*
		 * No change in engine... hence don't reprogram the
		 * engine, and don't change references.
		 */
		goto ok;
	}

	e->e_format = ENG_FORMAT(e);
	e->e_nchan = ENG_CHANNELS(e);
	e->e_rate = ENG_RATE(e);

	/* Select format converters for the engine. */
	switch (e->e_format) {
	case AUDIO_FORMAT_S24_NE:
		e->e_export = auimpl_export_24ne;
		e->e_import = auimpl_import_24ne;
		sampsz = 4;
		break;
	case AUDIO_FORMAT_S32_NE:
		e->e_export = auimpl_export_32ne;
		e->e_import = auimpl_import_32ne;
		sampsz = 4;
		break;
	case AUDIO_FORMAT_S24_OE:
		e->e_export = auimpl_export_24oe;
		e->e_import = auimpl_import_24oe;
		sampsz = 4;
		break;
	case AUDIO_FORMAT_S32_OE:
		e->e_export = auimpl_export_32oe;
		e->e_import = auimpl_import_32oe;
		sampsz = 4;
		break;
	case AUDIO_FORMAT_S16_NE:
		e->e_export = auimpl_export_16ne;
		e->e_import = auimpl_import_16ne;
		sampsz = 2;
		break;
	case AUDIO_FORMAT_S16_OE:
		e->e_export = auimpl_export_16oe;
		e->e_import = auimpl_import_16oe;
		sampsz = 2;
		break;
	case AUDIO_FORMAT_AC3:
		e->e_export = auimpl_export_24ne;
		e->e_import = auimpl_import_24ne;
		flags |= ENGINE_EXCLUSIVE;
		sampsz = 2;
		break;
	default:
		audio_dev_warn(d, "bad format");
		rv = ENOTSUP;
		goto done;
	}

	fragfr = e->e_rate / audio_intrhz;
	if ((fragfr > AUDIO_CHBUFS) || (fragfr < 1)) {
		audio_dev_warn(d, "invalid fragment configration");
		rv = EINVAL;
		goto done;
	}

	/* Sanity test a few values. */
	if ((e->e_nchan < 0) || (e->e_nchan > AUDIO_MAX_CHANNELS) ||
	    (e->e_rate < 5000) || (e->e_rate > 192000)) {
		audio_dev_warn(d, "bad engine channels or rate");
		rv = EINVAL;
		goto done;
	}

	if ((e->e_nframes <= (fragfr * 2)) || (e->e_data == NULL)) {
		audio_dev_warn(d, "improper engine configuration");
		rv = EINVAL;
		goto done;
	}

	e->e_framesz = e->e_nchan * sampsz;
	e->e_fragfr = fragfr;
	e->e_head = 0;
	e->e_tail = 0;
	e->e_hidx = 0;
	e->e_tidx = 0;
	e->e_limiter_state = 0x10000;
	bzero(e->e_data, e->e_nframes * e->e_framesz);

	if (e->e_ops.audio_engine_playahead == NULL) {
		e->e_playahead = (fragfr * 3) / 2;
	} else {
		e->e_playahead = ENG_PLAYAHEAD(e);
		/*
		 * Need to have at least a fragment plus some extra to
		 * avoid underruns.
		 */
		if (e->e_playahead < ((fragfr * 3) / 2)) {
			e->e_playahead = (fragfr * 3) / 2;
		}

		/*
		 * Impossible to queue more frames than FIFO can hold.
		 */
		if (e->e_playahead > e->e_nframes) {
			e->e_playahead = (fragfr * 3) / 2;
		}
	}

	for (i = 0; i < e->e_nchan; i++) {
		if (e->e_ops.audio_engine_chinfo == NULL) {
			e->e_choffs[i] = i;
			e->e_chincr[i] = e->e_nchan;
		} else {
			ENG_CHINFO(e, i, &e->e_choffs[i], &e->e_chincr[i]);
		}
	}

	e->e_flags |= flags;

	/*
	 * Arrange for the engine to be started.  We defer this to the
	 * periodic callback, to ensure that the start happens near
	 * the edge of the periodic callback.  This is necessary to
	 * ensure that the first fragment processed is about the same
	 * size as the usual fragment size.  (Basically, the problem
	 * is that we have only 10 msec resolution with the periodic
	 * interface, whch is rather unfortunate.)
	 */
	e->e_need_start = B_TRUE;

	if (flags & ENGINE_OUTPUT) {
		/*
		 * Start the output callback to populate the engine on
		 * startup.  This avoids a false underrun when we're
		 * first starting up.
		 */
		auimpl_output_preload(e);

		e->e_periodic = ddi_periodic_add(auimpl_output_callback, e,
		    NANOSEC / audio_intrhz, audio_priority);
	} else {
		e->e_periodic = ddi_periodic_add(auimpl_input_callback, e,
		    NANOSEC / audio_intrhz, audio_priority);
	}

ok:
	sp->s_phys_parms->p_rate = e->e_rate;
	sp->s_phys_parms->p_nchan = e->e_nchan;

	/* Configure the engine. */
	mutex_enter(&sp->s_lock);
	rv = auimpl_format_setup(sp, parms, mask);
	mutex_exit(&sp->s_lock);

done:
	mutex_exit(&e->e_lock);
	mutex_exit(&d->d_lock);

	return (rv);
}

void
auimpl_engine_close(audio_stream_t *sp)
{
	audio_engine_t	*e = sp->s_engine;
	audio_dev_t	*d;

	if (e == NULL)
		return;

	d = e->e_dev;

	mutex_enter(&d->d_lock);
	while (d->d_suspended) {
		cv_wait(&d->d_ctrl_cv, &d->d_lock);
	}

	mutex_enter(&e->e_lock);
	sp->s_engine = NULL;
	list_remove(&e->e_streams, sp);
	if (list_is_empty(&e->e_streams)) {
		ENG_STOP(e);
		ddi_periodic_delete(e->e_periodic);
		e->e_periodic = 0;
		e->e_flags &= ENGINE_DRIVER_FLAGS;
		ENG_CLOSE(e);
	}
	mutex_exit(&e->e_lock);

	cv_broadcast(&d->d_cv);
	mutex_exit(&d->d_lock);
}

int
audio_dev_register(audio_dev_t *d)
{
	list_t *l;
	audio_dev_t *srch;
	int start;

	/*
	 * Make sure we don't automatically unload.  This prevents
	 * loss of hardware settings when no audio clients are
	 * running.
	 */
	(void) ddi_prop_update_int(DDI_DEV_T_NONE, d->d_dip,
	    DDI_NO_AUTODETACH, 1);

	/*
	 * This does an in-order insertion, finding the first available
	 * free index.  "Special" devices (ones without any actual engines)
	 * are all numbered 0.  There should only be one of them anyway.
	 * All others start at one.
	 */
	if (d->d_flags & DEV_SNDSTAT_CAP) {
		start = 0;
	} else {
		start = 1;
	}
	d->d_index = start;

	rw_enter(&auimpl_dev_lock, RW_WRITER);
	l = &auimpl_devs_by_index;
	for (srch = list_head(l); srch; srch = list_next(l, srch)) {
		/* skip over special nodes */
		if (srch->d_index < start)
			continue;
		if (srch->d_index > d->d_index) {
			/* found a free spot! */
			break;
		}
		d->d_index++;
	}
	/*
	 * NB: If srch is NULL, then list_insert_before puts
	 * it on the tail of the list.  So if we didn't find a
	 * hole, then that's where we want it.
	 */
	list_insert_before(l, srch, d);

	/* insert in order by number */
	l = &auimpl_devs_by_number;
	for (srch = list_head(l); srch; srch = list_next(l, srch)) {
		if (srch->d_number >= d->d_number) {
			break;
		}
	}
	list_insert_before(l, srch, d);

	rw_exit(&auimpl_dev_lock);

	if (auimpl_create_minors(d) != 0) {
		rw_enter(&auimpl_dev_lock, RW_WRITER);
		auimpl_remove_minors(d);
		list_remove(&auimpl_devs_by_index, d);
		list_remove(&auimpl_devs_by_number, d);
		rw_exit(&auimpl_dev_lock);
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

int
audio_dev_unregister(audio_dev_t *d)
{
	rw_enter(&auimpl_dev_lock, RW_WRITER);

	mutex_enter(&d->d_lock);
	/* if we are still in use, we can't unregister */
	if (d->d_refcnt) {
		mutex_exit(&d->d_lock);
		rw_exit(&auimpl_dev_lock);
		return (DDI_FAILURE);
	}
	auimpl_remove_minors(d);
	list_remove(&auimpl_devs_by_index, d);
	list_remove(&auimpl_devs_by_number, d);
	mutex_exit(&d->d_lock);

	rw_exit(&auimpl_dev_lock);

	return (DDI_SUCCESS);
}

static int
auimpl_engine_ksupdate(kstat_t *ksp, int rw)
{
	audio_engine_t *e = ksp->ks_private;
	struct audio_stats *st = &e->e_stats;

	if (rw == KSTAT_WRITE) {
		return (EACCES);
	}

	mutex_enter(&e->e_lock);
	st->st_head.value.ui64 = e->e_head;
	st->st_tail.value.ui64 = e->e_tail;
	st->st_flags.value.ui32 = e->e_flags;
	st->st_nbytes.value.ui32 = e->e_framesz * e->e_nframes;
	st->st_framesz.value.ui32 = e->e_framesz;
	st->st_hidx.value.ui32 = e->e_hidx;
	st->st_tidx.value.ui32 = e->e_tidx;
	st->st_format.value.ui32 = e->e_format;
	st->st_nchan.value.ui32 = e->e_nchan;
	st->st_rate.value.ui32 = e->e_rate;
	st->st_errors.value.ui32 = e->e_errors;
	st->st_engine_underruns.value.ui32 = e->e_underruns;
	st->st_engine_overruns.value.ui32 = e->e_overruns;
	st->st_stream_underruns.value.ui32 = e->e_stream_underruns;
	st->st_stream_overruns.value.ui32 = e->e_stream_overruns;
	st->st_suspended.value.ui32 = e->e_suspended;
	st->st_failed.value.ui32 = e->e_failed;
	st->st_playahead.value.ui32 = e->e_playahead;
	mutex_exit(&e->e_lock);

	return (0);
}

static void
auimpl_engine_ksinit(audio_dev_t *d, audio_engine_t *e)
{
	char			name[32];
	struct audio_stats	*st;

	(void) snprintf(name, sizeof (name), "engine_%d", e->e_num);

	e->e_ksp = kstat_create(ddi_driver_name(d->d_dip), d->d_instance,
	    name, "misc", KSTAT_TYPE_NAMED,
	    sizeof (struct audio_stats) / sizeof (kstat_named_t), 0);

	if (e->e_ksp == NULL) {
		audio_dev_warn(d, "unable to initialize kstats");
		return;
	}

	st = &e->e_stats;
	e->e_ksp->ks_data = st;
	e->e_ksp->ks_private = e;
	e->e_ksp->ks_lock = NULL;
	e->e_ksp->ks_update = auimpl_engine_ksupdate;
	kstat_named_init(&st->st_head, "head", KSTAT_DATA_UINT64);
	kstat_named_init(&st->st_tail, "tail", KSTAT_DATA_UINT64);
	kstat_named_init(&st->st_flags, "flags", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_nbytes, "nbytes", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_framesz, "framesz", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_hidx, "hidx", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_tidx, "tidx", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_format, "format", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_nchan, "channels", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_rate, "rate", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_errors, "errors", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_engine_overruns, "engine_overruns",
	    KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_engine_underruns, "engine_underruns",
	    KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_stream_overruns, "stream_overruns",
	    KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_stream_underruns, "stream_underruns",
	    KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_playahead, "playahead", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_suspended, "suspended", KSTAT_DATA_UINT32);
	kstat_named_init(&st->st_failed, "failed", KSTAT_DATA_UINT32);
	kstat_install(e->e_ksp);
}

void
audio_dev_add_engine(audio_dev_t *d, audio_engine_t *e)
{
	mutex_enter(&d->d_lock);

	e->e_num = d->d_engno++;

	auimpl_engine_ksinit(d, e);

	/* check for duplex */
	if ((e->e_flags & ENGINE_OUTPUT_CAP) && (d->d_flags & DEV_INPUT_CAP)) {
		d->d_flags |= DEV_DUPLEX_CAP;
	}
	if ((e->e_flags & ENGINE_INPUT_CAP) && (d->d_flags & DEV_OUTPUT_CAP)) {
		d->d_flags |= DEV_DUPLEX_CAP;
	}
	/* add in the direction caps -- must be done after duplex above */
	if (e->e_flags & ENGINE_OUTPUT_CAP) {
		d->d_flags |= DEV_OUTPUT_CAP;
	}
	if (e->e_flags & ENGINE_INPUT_CAP) {
		d->d_flags |= DEV_INPUT_CAP;
	}

	list_insert_tail(&d->d_engines, e);
	e->e_dev = d;
	mutex_exit(&d->d_lock);
}

void
audio_dev_remove_engine(audio_dev_t *d, audio_engine_t *e)
{
	mutex_enter(&d->d_lock);
	list_remove(&d->d_engines, e);
	e->e_dev = NULL;
	if (e->e_ksp)
		kstat_delete(e->e_ksp);
	e->e_ksp = NULL;
	mutex_exit(&d->d_lock);
}

/*
 * Change the number.
 */
void
auclnt_set_dev_number(audio_dev_t *d, int num)
{
	list_t		*l = &auimpl_devs_by_number;
	audio_dev_t	*srch;

	/* reorder our list */
	rw_enter(&auimpl_dev_lock, RW_WRITER);
	d->d_number = num;
	list_remove(l, d);
	for (srch = list_head(l); srch; srch = list_next(l, srch)) {
		if (srch->d_number >= d->d_number) {
			break;
		}
	}
	list_insert_before(l, srch, d);

	rw_exit(&auimpl_dev_lock);
}

void
auclnt_walk_devs(int (*walker)(audio_dev_t *, void *), void *arg)
{
	audio_dev_t	*d;
	boolean_t	cont;
	list_t		*l;

	l = &auimpl_devs_by_index;
	rw_enter(&auimpl_dev_lock, RW_READER);
	for (d = list_head(l); d; d = list_next(l, d)) {
		cont = walker(d, arg);
		if (cont == AUDIO_WALK_STOP)
			break;
	}
	rw_exit(&auimpl_dev_lock);
}

void
auclnt_walk_devs_by_number(int (*walker)(audio_dev_t *, void *), void *arg)
{
	audio_dev_t	*d;
	boolean_t	cont;
	list_t		*l;

	l = &auimpl_devs_by_number;
	rw_enter(&auimpl_dev_lock, RW_READER);
	for (d = list_head(l); d; d = list_next(l, d)) {
		cont = walker(d, arg);
		if (cont == AUDIO_WALK_STOP)
			break;
	}
	rw_exit(&auimpl_dev_lock);
}

void
auclnt_dev_walk_engines(audio_dev_t *d,
    int (*walker)(audio_engine_t *, void *),
    void *arg)
{
	audio_engine_t *e;
	list_t *l = &d->d_engines;

	mutex_enter(&d->d_lock);
	for (e = list_head(l); e != NULL; e = list_next(l, e)) {
		if (walker(e, arg) == AUDIO_WALK_STOP) {
			break;
		}
	}
	mutex_exit(&d->d_lock);
}

int
auclnt_engine_get_format(audio_engine_t *e)
{
	return (ENG_FORMAT(e));
}

int
auclnt_engine_get_channels(audio_engine_t *e)
{
	return (ENG_CHANNELS(e));
}

int
auclnt_engine_get_rate(audio_engine_t *e)
{
	return (ENG_RATE(e));
}

uint_t
auclnt_engine_get_capab(audio_engine_t *e)
{
	uint_t capab = 0;

	if (e->e_flags & ENGINE_INPUT_CAP) {
		capab |= AUDIO_CLIENT_CAP_RECORD;
	}
	if (e->e_flags & ENGINE_OUTPUT_CAP) {
		capab |= AUDIO_CLIENT_CAP_PLAY;
	}
	return (capab);
}

/*
 * This function suspends an engine.  The intent is to pause the
 * engine temporarily so that it does not underrun while user threads
 * are suspended.  The driver is still responsible for actually doing
 * the driver suspend work -- all this does is put the engine in a
 * paused state.  It does not prevent, for example, threads from
 * accessing the hardware.
 *
 * A properly implemented driver won't even be aware of the existence
 * of this routine -- the driver will just handle the suspend &
 * resume.  At the point of suspend & resume, the driver will see that
 * the engines are not running (as if all threads had "paused" it).
 *
 * Failure to execute either of the routines below is not critical,
 * but will probably lead to underruns and overflows as the kernel
 * driver gets resumed well in advance of the time when user threads
 * are ready to start operation.
 */
static void
auimpl_engine_suspend(audio_engine_t *e)
{
	ASSERT(mutex_owned(&e->e_lock));

	if (e->e_failed || e->e_suspended) {
		e->e_suspended = B_TRUE;
		return;
	}
	e->e_suspended = B_TRUE;
	if (e->e_flags & ENGINE_INPUT) {
		e->e_head = ENG_COUNT(e);
		ENG_STOP(e);
	}
	if (e->e_flags & ENGINE_OUTPUT) {
		e->e_tail = ENG_COUNT(e);
		ENG_STOP(e);
	}
}

static void
auimpl_engine_resume(audio_engine_t *e)
{
	ASSERT(mutex_owned(&e->e_lock));
	ASSERT(e->e_suspended);

	if (e->e_failed) {
		/* No longer suspended, but still failed! */
		e->e_suspended = B_FALSE;
		return;
	}

	if (e->e_flags & (ENGINE_INPUT | ENGINE_OUTPUT)) {

		auimpl_engine_reset(e);

		if (e->e_flags & ENGINE_OUTPUT) {
			auimpl_output_preload(e);
		}

		e->e_need_start = B_TRUE;
	}
	e->e_suspended = B_FALSE;
	cv_broadcast(&e->e_cv);
}

static int
auimpl_dev_suspend(audio_dev_t *d, void *dontcare)
{
	list_t		*l;
	audio_engine_t	*e;

	_NOTE(ARGUNUSED(dontcare));

	mutex_enter(&d->d_lock);
	mutex_enter(&d->d_ctrl_lock);
	if (d->d_suspended) {
		d->d_suspended++;
		mutex_exit(&d->d_ctrl_lock);
		mutex_exit(&d->d_lock);
		return (AUDIO_WALK_CONTINUE);
	}

	d->d_suspended++;

	(void) auimpl_save_controls(d);
	mutex_exit(&d->d_ctrl_lock);

	l = &d->d_engines;
	for (e = list_head(l); e != NULL; e = list_next(l, e)) {
		mutex_enter(&e->e_lock);
		auimpl_engine_suspend(e);
		mutex_exit(&e->e_lock);
	}
	mutex_exit(&d->d_lock);

	return (AUDIO_WALK_CONTINUE);
}

static int
auimpl_dev_resume(audio_dev_t *d, void *dontcare)
{
	list_t		*l;
	audio_engine_t	*e;

	_NOTE(ARGUNUSED(dontcare));

	mutex_enter(&d->d_lock);
	mutex_enter(&d->d_ctrl_lock);

	ASSERT(d->d_suspended);
	d->d_suspended--;
	if (d->d_suspended) {
		mutex_exit(&d->d_ctrl_lock);
		mutex_exit(&d->d_lock);
		return (AUDIO_WALK_CONTINUE);
	}

	(void) auimpl_restore_controls(d);
	cv_broadcast(&d->d_ctrl_cv);
	mutex_exit(&d->d_ctrl_lock);

	l = &d->d_engines;
	for (e = list_head(l); e != NULL; e = list_next(l, e)) {
		mutex_enter(&e->e_lock);
		auimpl_engine_resume(e);
		mutex_exit(&e->e_lock);
	}
	mutex_exit(&d->d_lock);

	return (AUDIO_WALK_CONTINUE);
}

boolean_t
auimpl_cpr(void *arg, int code)
{
	_NOTE(ARGUNUSED(arg));

	switch (code) {
	case CB_CODE_CPR_CHKPT:
		auclnt_walk_devs(auimpl_dev_suspend, NULL);
		return (B_TRUE);

	case CB_CODE_CPR_RESUME:
		auclnt_walk_devs(auimpl_dev_resume, NULL);
		return (B_TRUE);

	default:
		return (B_FALSE);
	}
}

void
audio_dev_suspend(audio_dev_t *d)
{
	(void) auimpl_dev_suspend(d, NULL);
}

void
audio_dev_resume(audio_dev_t *d)
{
	(void) auimpl_dev_resume(d, NULL);
}

static callb_id_t	auimpl_cpr_id = 0;

void
auimpl_dev_init(void)
{
	rw_init(&auimpl_dev_lock, NULL, RW_DRIVER, NULL);
	list_create(&auimpl_devs_by_index, sizeof (struct audio_dev),
	    offsetof(struct audio_dev, d_by_index));
	list_create(&auimpl_devs_by_number, sizeof (struct audio_dev),
	    offsetof(struct audio_dev, d_by_number));

	/*
	 * We "borrow" the CB_CL_CPR_PM class, which gets executed at
	 * about the right time for us.  It would be nice to have a
	 * new CB_CL_CPR_AUDIO class, but it isn't critical at this
	 * point.
	 *
	 * Note that we don't care about our thread id.
	 */
	auimpl_cpr_id = callb_add(auimpl_cpr, NULL, CB_CL_CPR_PM, "audio_cpr");
}

void
auimpl_dev_fini(void)
{
	(void) callb_delete(auimpl_cpr_id);
	list_destroy(&auimpl_devs_by_index);
	list_destroy(&auimpl_devs_by_number);
	rw_destroy(&auimpl_dev_lock);
}

void
audio_engine_set_private(audio_engine_t *eng, void *prv)
{
	eng->e_private = prv;
}

void *
audio_engine_get_private(audio_engine_t *eng)
{
	return (eng->e_private);
}

void
audio_dump_bytes(const uint8_t *w, int dcount)
{
	char		line[64];
	char		*s;
	int		i;
	const int	wrap = 16;

	s = line;
	line[0] = 0;

	cmn_err(CE_NOTE, "starting @ %p", (void *)w);
	for (i = 0; i < dcount; i++) {

		(void) sprintf(s, " %02x", *w);
		s += strlen(s);
		w++;

		if ((i % wrap) == (wrap - 1)) {
			cmn_err(CE_NOTE, "%08x:%s", i - (wrap - 1), line);
			line[0] = 0;
			s = line;
		}
	}

	if ((i % wrap) != 0) {
		cmn_err(CE_NOTE, "%08x:%s", i - (i % wrap), line);
	}
}

void
audio_dump_words(const uint16_t *w, int dcount)
{
	char		line[64];
	char		*s;
	int		i;
	const int	wrap = 8;

	s = line;
	line[0] = 0;

	cmn_err(CE_NOTE, "starting @ %p", (void *)w);
	for (i = 0; i < dcount; i++) {

		(void) sprintf(s, " %04x", *w);
		s += strlen(s);
		w++;

		if ((i % wrap) == (wrap - 1)) {
			cmn_err(CE_NOTE, "%08x:%s", i - (wrap - 1), line);
			line[0] = 0;
			s = line;
		}
	}

	if ((i % wrap) != 0) {
		cmn_err(CE_NOTE, "%08x:%s", i - (i % wrap), line);
	}
}

void
audio_dump_dwords(const uint32_t *w, int dcount)
{
	char		line[128];
	char		*s;
	int		i;
	const int	wrap = 4;

	s = line;
	line[0] = 0;

	cmn_err(CE_NOTE, "starting @ %p", (void *)w);
	for (i = 0; i < dcount; i++) {

		(void) sprintf(s, " %08x", *w);
		s += strlen(s);
		w++;

		if ((i % wrap) == (wrap - 1)) {
			cmn_err(CE_NOTE, "%08x:%s", i - (wrap - 1), line);
			line[0] = 0;
			s = line;
		}
	}

	if ((i % wrap) != 0) {
		cmn_err(CE_NOTE, "%08x:%s", i - (i % wrap), line);
	}
}