xref: /titanic_51/usr/src/uts/common/os/ndifm.c (revision b9bd317cda1afb3a01f4812de73e8cec888cbbd7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * Fault Management for Nexus Device Drivers
30  *
31  * In addition to implementing and supporting Fault Management for Device
32  * Drivers (ddifm.c), nexus drivers must support their children by
33  * reporting FM capabilities, intializing interrupt block cookies
34  * for error handling callbacks and caching mapped resources for lookup
35  * during the detection of an IO transaction error.
36  *
37  * It is typically the nexus driver that receives an error indication
38  * for a fault that may have occurred in the data path of an IO transaction.
39  * Errors may be detected or received via an interrupt, a callback from
40  * another subsystem (e.g. a cpu trap) or examination of control data.
41  *
42  * Upon detection of an error, the nexus has a responsibility to alert
43  * its children of the error and the transaction associated with that
44  * error.  The actual implementation may vary depending upon the capabilities
45  * of the nexus, its underlying hardware and its children.  In this file,
46  * we provide support for typical nexus driver fault management tasks.
47  *
48  * Fault Management Initialization
49  *
50  *      Nexus drivers must implement two new busops, bus_fm_init() and
51  *      bus_fm_fini().  bus_fm_init() is called from a child nexus or device
52  *      driver and is expected to initialize any per-child state and return
53  *      the FM and error interrupt priority levels of the nexus driver.
54  *      Similarly, bus_fm_fini() is called by child drivers and should
55  *      clean-up any resources allocated during bus_fm_init().
56  *      These functions are called from passive kernel context, typically from
57  *      driver attach(9F) and detach(9F) entry points.
58  *
59  * Error Handler Dispatching
60  *
61  *      Nexus drivers implemented to support error handler capabilities
62  *	should invoke registered error handler callbacks for child drivers
63  *	thought to be involved in the error.
64  *	ndi_fm_handler_dispatch() is used to invoke
65  *      all error handlers and returns one of the following status
66  *      indications:
67  *
68  *      DDI_FM_OK - No errors found by any child
69  *      DDI_FM_FATAL - one or more children have detected a fatal error
70  *      DDI_FM_NONFATAL - no fatal errors, but one or more children have
71  *                            detected a non-fatal error
72  *
73  *      ndi_fm_handler_dispatch() may be called in any context
74  *      subject to the constraints specified by the interrupt iblock cookie
75  *      returned during initialization.
76  *
77  * Protected Accesses
78  *
79  *      When an access handle is mapped or a DMA handle is bound via the
80  *      standard busops, bus_map() or bus_dma_bindhdl(), a child driver
81  *      implemented to support DDI_FM_ACCCHK_CAPABLE or
82  *	DDI_FM_DMACHK_CAPABLE capabilites
83  *	expects the nexus to flag any errors detected for transactions
84  *	associated with the mapped or bound handles.
85  *
86  *      Children nexus or device drivers will set the following flags
87  *      in their ddi_device_access or dma_attr_flags when requesting
88  *      the an access or DMA handle mapping:
89  *
90  *      DDI_DMA_FLAGERR - nexus should set error status for any errors
91  *                              detected for a failed DMA transaction.
92  *      DDI_ACC_FLAGERR - nexus should set error status for any errors
93  *                              detected for a failed PIO transaction.
94  *
95  *      A nexus is expected to provide additional error detection and
96  *      handling for handles with these flags set.
97  *
98  * Exclusive Bus Access
99  *
100  *      In cases where a driver requires a high level of fault tolerance
101  *      for a programmed IO transaction, it is neccessary to grant exclusive
102  *      access to the bus resource.  Exclusivity guarantees that a fault
103  *      resulting from a transaction on the bus can be easily traced and
104  *      reported to the driver requesting the transaction.
105  *
106  *      Nexus drivers must implement two new busops to support exclusive
107  *      access, bus_fm_access_enter() and bus_fm_access_exit().  The IO
108  *      framework will use these functions when it must set-up access
109  *      handles that set devacc_attr_access to DDI_ACC_CAUTIOUS in
110  *      their ddi_device_acc_attr_t request.
111  *
112  *      Upon receipt of a bus_fm_access_enter() request, the nexus must prevent
113  *      all other access requests until it receives bus_fm_access_exit()
114  *      for the requested bus instance. bus_fm_access_enter() and
115  *	bus_fm_access_exit() may be called from user, kernel or kernel
116  *	interrupt context.
117  *
118  * Access and DMA Handle Caching
119  *
120  *      To aid a nexus driver in associating access or DMA handles with
121  *      a detected error, the nexus should cache all handles that are
122  *      associated with DDI_ACC_FLAGERR, DDI_ACC_CAUTIOUS_ACC or
123  *	DDI_DMA_FLAGERR requests from its children.  ndi_fmc_insert() is
124  *	called by a nexus to cache handles with the above protection flags
125  *	and ndi_fmc_remove() is called when that handle is unmapped or
126  *	unbound by the requesting child.  ndi_fmc_insert() and
127  *	ndi_fmc_remove() may be called from any user or kernel context.
128  *
129  *	FM caches are allocated during ddi_fm_init() and maintained
130  *	as an array of elements that may be on one of two lists:
131  *	free or active.  The free list is a singly-linked list of
132  *	elements available for activity.  ndi_fm_insert() moves the
133  *	element at the head of the free to the active list.  The active
134  *	list is a doubly-linked searchable list.
135  *	When a handle is unmapped or unbound, its associated cache
136  *	entry is removed from the active list back to the free list.
137  *
138  *      Upon detection of an error, the nexus may invoke ndi_fmc_error() to
139  *      iterate over the handle cache of one or more of its FM compliant
140  *      children.  A comparison callback function is provided upon each
141  *      invocation of ndi_fmc_error() to tell the IO framework if a
142  *      handle is associated with an error.  If so, the framework will
143  *      set the error status for that handle before returning from
144  *      ndi_fmc_error().
145  *
146  *      ndi_fmc_error() may be called in any context
147  *      subject to the constraints specified by the interrupt iblock cookie
148  *      returned during initialization of the nexus and its children.
149  *
150  */
151 
152 #include <sys/types.h>
153 #include <sys/param.h>
154 #include <sys/debug.h>
155 #include <sys/sunddi.h>
156 #include <sys/sunndi.h>
157 #include <sys/ddi.h>
158 #include <sys/ndi_impldefs.h>
159 #include <sys/devctl.h>
160 #include <sys/nvpair.h>
161 #include <sys/ddifm.h>
162 #include <sys/ndifm.h>
163 #include <sys/spl.h>
164 #include <sys/sysmacros.h>
165 #include <sys/devops.h>
166 #include <sys/atomic.h>
167 #include <sys/fm/io/ddi.h>
168 
169 /*
170  * Allocate and initialize a fault management resource cache
171  * A fault management cache consists of a set of cache elements that
172  * may be on one of two lists: free or active.
173  *
174  * At creation time, every element but one is placed on the free list
175  * except for the first element.  This element is reserved as the first
176  * element of the active list and serves as an anchor for the active
177  * list in ndi_fmc_insert() and ndi_fmc_remove().  In these functions,
178  * it is not neccessary to check for the existence or validity of
179  * the active list.
180  */
181 void
182 i_ndi_fmc_create(ndi_fmc_t **fcpp, int qlen, ddi_iblock_cookie_t ibc)
183 {
184 	ndi_fmc_t *fcp;
185 	ndi_fmcentry_t *fep;
186 
187 	ASSERT(qlen > 1);
188 
189 	fcp = kmem_zalloc(sizeof (ndi_fmc_t), KM_SLEEP);
190 	mutex_init(&fcp->fc_lock, NULL, MUTEX_DRIVER, ibc);
191 	mutex_init(&fcp->fc_free_lock, NULL, MUTEX_DRIVER, NULL);
192 
193 	/* Preallocate and initialize entries for this fm cache */
194 	fcp->fc_elems = kmem_zalloc(qlen * sizeof (ndi_fmcentry_t), KM_SLEEP);
195 
196 	fcp->fc_len = qlen;
197 
198 	/* Intialize the active and free lists */
199 	fcp->fc_active = fcp->fc_tail = fcp->fc_elems;
200 	fcp->fc_free = fcp->fc_elems + 1;
201 	qlen--;
202 	for (fep = fcp->fc_free; qlen > 1; qlen--) {
203 		fep->fce_prev = fep + 1;
204 		fep++;
205 	}
206 
207 	*fcpp = fcp;
208 }
209 
210 /*
211  * Destroy and resources associated with the given fault management cache.
212  */
213 void
214 i_ndi_fmc_destroy(ndi_fmc_t *fcp)
215 {
216 	if (fcp == NULL)
217 		return;
218 
219 	kmem_free(fcp->fc_elems, fcp->fc_len * sizeof (ndi_fmcentry_t));
220 	kmem_free(fcp, sizeof (ndi_fmc_t));
221 }
222 
223 /*
224  * Grow an existing fault management cache by grow_sz number of entries
225  */
226 static int
227 fmc_grow(ndi_fmc_t *fcp, int flag, int grow_sz)
228 {
229 	int olen, nlen;
230 	void *resource;
231 	ndi_fmcentry_t *ncp, *oep, *nep, *nnep;
232 
233 	ASSERT(grow_sz);
234 	ASSERT(MUTEX_HELD(&fcp->fc_free_lock));
235 
236 	/* Allocate a new cache */
237 	nlen = grow_sz + fcp->fc_len;
238 	if ((ncp = kmem_zalloc(nlen * sizeof (ndi_fmcentry_t),
239 	    KM_NOSLEEP)) == NULL)
240 		return (1);
241 
242 	/* Migrate old cache to new cache */
243 	oep = fcp->fc_elems;
244 	olen = fcp->fc_len;
245 	for (nep = ncp; ; olen--) {
246 		resource = nep->fce_resource = oep->fce_resource;
247 		nep->fce_bus_specific = oep->fce_bus_specific;
248 		if (resource) {
249 			if (flag == DMA_HANDLE) {
250 				((ddi_dma_impl_t *)resource)->
251 				    dmai_error.err_fep = nep;
252 			} else if (flag == ACC_HANDLE) {
253 				((ddi_acc_impl_t *)resource)->
254 				    ahi_err->err_fep = nep;
255 			}
256 		}
257 
258 		/*
259 		 * This is the last entry.  Set the tail pointer and
260 		 * terminate processing of the old cache.
261 		 */
262 		if (olen == 1) {
263 			fcp->fc_tail = nep;
264 			++nep;
265 			break;
266 		}
267 
268 		/*
269 		 * Set the next and previous pointer for the new cache
270 		 * entry.
271 		 */
272 		nnep = nep + 1;
273 		nep->fce_next = nnep;
274 		nnep->fce_prev = nep;
275 
276 		/* Advance to the next entry */
277 		++oep;
278 		nep = nnep;
279 	}
280 
281 	/* Initialize and add remaining new cache entries to the free list */
282 	for (fcp->fc_free = nep; nlen > fcp->fc_len + 1; nlen--) {
283 		nep->fce_prev = nep + 1;
284 		nep++;
285 	}
286 
287 	oep = fcp->fc_elems;
288 	olen = fcp->fc_len;
289 	nlen = grow_sz + olen;
290 
291 	/*
292 	 * Update the FM cache array and active list pointers.
293 	 * Updates to these pointers require us to acquire the
294 	 * FMA cache lock to prevent accesses to a stale active
295 	 * list in ndi_fmc_error().
296 	 */
297 
298 	mutex_enter(&fcp->fc_lock);
299 	fcp->fc_active = ncp;
300 	fcp->fc_elems = ncp;
301 	fcp->fc_len = nlen;
302 	mutex_exit(&fcp->fc_lock);
303 
304 	kmem_free(oep, olen * sizeof (ndi_fmcentry_t));
305 
306 	return (0);
307 }
308 
309 /*
310  * ndi_fmc_insert -
311  * 	Add a new entry to the specified cache.
312  *
313  * 	This function must be called at or below LOCK_LEVEL
314  */
315 void
316 ndi_fmc_insert(dev_info_t *dip, int flag, void *resource, void *bus_specific)
317 {
318 	struct dev_info *devi = DEVI(dip);
319 	ndi_fmc_t *fcp;
320 	ndi_fmcentry_t *fep, **fpp;
321 	struct i_ddi_fmhdl *fmhdl;
322 
323 	ASSERT(devi);
324 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
325 
326 	fmhdl = devi->devi_fmhdl;
327 	if (fmhdl == NULL) {
328 		i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL, DDI_NOSLEEP);
329 		return;
330 	}
331 
332 	if (flag == DMA_HANDLE) {
333 		if (!DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
334 			i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL,
335 			    DDI_NOSLEEP);
336 			return;
337 		}
338 		fcp = fmhdl->fh_dma_cache;
339 		fpp = &((ddi_dma_impl_t *)resource)->dmai_error.err_fep;
340 	} else if (flag == ACC_HANDLE) {
341 		if (!DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
342 			i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL,
343 			    DDI_NOSLEEP);
344 			return;
345 		}
346 		fcp = fmhdl->fh_acc_cache;
347 		fpp = &((ddi_acc_impl_t *)resource)->ahi_err->err_fep;
348 	}
349 	ASSERT(*fpp == NULL);
350 
351 	mutex_enter(&fcp->fc_free_lock);
352 
353 	/* Get an entry from the free list */
354 	fep = fcp->fc_free;
355 	if (fep == NULL) {
356 		if (fmc_grow(fcp, flag,
357 		    (flag == ACC_HANDLE ? default_acccache_sz :
358 		    default_dmacache_sz)) != 0) {
359 
360 			/* Unable to get an entry or grow this cache */
361 			atomic_add_64(
362 			    &fmhdl->fh_kstat.fek_fmc_full.value.ui64, 1);
363 			mutex_exit(&fcp->fc_free_lock);
364 			return;
365 		}
366 		atomic_add_64(&fmhdl->fh_kstat.fek_fmc_grew.value.ui64, 1);
367 		fep = fcp->fc_free;
368 	}
369 	fcp->fc_free = fep->fce_prev;
370 
371 	/*
372 	 * Set-up the handle resource and bus_specific information.
373 	 * Also remember the pointer back to the cache for quick removal.
374 	 */
375 	fep->fce_bus_specific = bus_specific;
376 	fep->fce_resource = resource;
377 	fep->fce_next = NULL;
378 	*fpp = fep;
379 
380 	/* Add entry to the end of the active list */
381 	mutex_enter(&fcp->fc_lock);
382 	fep->fce_prev = fcp->fc_tail;
383 	fcp->fc_tail->fce_next = fep;
384 	fcp->fc_tail = fep;
385 	mutex_exit(&fcp->fc_lock);
386 	mutex_exit(&fcp->fc_free_lock);
387 }
388 
389 /*
390  * 	Remove an entry from the specified cache of access or dma mappings
391  *
392  * 	This function must be called at or below LOCK_LEVEL.
393  */
394 void
395 ndi_fmc_remove(dev_info_t *dip, int flag, const void *resource)
396 {
397 	ndi_fmc_t *fcp;
398 	ndi_fmcentry_t *fep;
399 	struct dev_info *devi = DEVI(dip);
400 	struct i_ddi_fmhdl *fmhdl;
401 
402 	ASSERT(devi);
403 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
404 
405 	fmhdl = devi->devi_fmhdl;
406 	if (fmhdl == NULL) {
407 		i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL, DDI_NOSLEEP);
408 		return;
409 	}
410 
411 	/* Find cache entry pointer for this resource */
412 	if (flag == DMA_HANDLE) {
413 		if (!DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
414 			i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL,
415 			    DDI_NOSLEEP);
416 			return;
417 		}
418 		fcp = fmhdl->fh_dma_cache;
419 
420 		ASSERT(fcp);
421 
422 		mutex_enter(&fcp->fc_free_lock);
423 		fep = ((ddi_dma_impl_t *)resource)->dmai_error.err_fep;
424 		((ddi_dma_impl_t *)resource)->dmai_error.err_fep = NULL;
425 	} else if (flag == ACC_HANDLE) {
426 		if (!DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
427 			i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL,
428 			    DDI_NOSLEEP);
429 			return;
430 		}
431 		fcp = fmhdl->fh_acc_cache;
432 
433 		ASSERT(fcp);
434 
435 		mutex_enter(&fcp->fc_free_lock);
436 		fep = ((ddi_acc_impl_t *)resource)->ahi_err->err_fep;
437 		((ddi_acc_impl_t *)resource)->ahi_err->err_fep = NULL;
438 	} else {
439 		return;
440 	}
441 
442 	/*
443 	 * Resource not in cache, return
444 	 */
445 	if (fep == NULL) {
446 		mutex_exit(&fcp->fc_free_lock);
447 		return;
448 	}
449 
450 	/*
451 	 * Updates to FM cache pointers require us to grab fmc_lock
452 	 * to synchronize access to the cache for ndi_fmc_insert()
453 	 * and ndi_fmc_error()
454 	 */
455 	mutex_enter(&fcp->fc_lock);
456 	fep->fce_prev->fce_next = fep->fce_next;
457 	if (fep == fcp->fc_tail)
458 		fcp->fc_tail = fep->fce_prev;
459 	else
460 		fep->fce_next->fce_prev = fep->fce_prev;
461 	mutex_exit(&fcp->fc_lock);
462 
463 	/* Add entry back to the free list */
464 	fep->fce_prev = fcp->fc_free;
465 	fcp->fc_free = fep;
466 	mutex_exit(&fcp->fc_free_lock);
467 }
468 
469 int
470 ndi_fmc_entry_error(dev_info_t *dip, int flag, ddi_fm_error_t *derr,
471     const void *bus_err_state)
472 {
473 	int status, fatal = 0, nonfatal = 0;
474 	ndi_fmc_t *fcp = NULL;
475 	ndi_fmcentry_t *fep;
476 	struct i_ddi_fmhdl *fmhdl;
477 
478 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
479 
480 	fmhdl = DEVI(dip)->devi_fmhdl;
481 	ASSERT(fmhdl);
482 	status = DDI_FM_UNKNOWN;
483 
484 	if (flag == DMA_HANDLE && DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
485 		fcp = fmhdl->fh_dma_cache;
486 		ASSERT(fcp);
487 	} else if (flag == ACC_HANDLE && DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
488 		fcp = fmhdl->fh_acc_cache;
489 		ASSERT(fcp);
490 	}
491 
492 	if (fcp != NULL) {
493 
494 		/*
495 		 * Check active resource entries
496 		 */
497 		mutex_enter(&fcp->fc_lock);
498 		for (fep = fcp->fc_active->fce_next; fep != NULL;
499 		    fep = fep->fce_next) {
500 			ddi_fmcompare_t compare_func;
501 
502 			/*
503 			 * Compare captured error state with handle
504 			 * resources.  During the comparison and
505 			 * subsequent error handling, we block
506 			 * attempts to free the cache entry.
507 			 */
508 			compare_func = (flag == ACC_HANDLE) ?
509 			    i_ddi_fm_acc_err_cf_get((ddi_acc_handle_t)
510 			    fep->fce_resource) :
511 			    i_ddi_fm_dma_err_cf_get((ddi_dma_handle_t)
512 			    fep->fce_resource);
513 
514 			status = compare_func(dip, fep->fce_resource,
515 			    bus_err_state, fep->fce_bus_specific);
516 			if (status == DDI_FM_UNKNOWN || status == DDI_FM_OK)
517 				continue;
518 
519 			if (status == DDI_FM_FATAL)
520 				++fatal;
521 			else if (status == DDI_FM_NONFATAL)
522 				++nonfatal;
523 
524 			/* Set the error for this resource handle */
525 			if (flag == ACC_HANDLE) {
526 				ddi_acc_handle_t ap = fep->fce_resource;
527 
528 				i_ddi_fm_acc_err_set(ap, derr->fme_ena, status,
529 				    DDI_FM_ERR_UNEXPECTED);
530 				ddi_fm_acc_err_get(ap, derr, DDI_FME_VERSION);
531 				derr->fme_acc_handle = ap;
532 			} else {
533 				ddi_dma_handle_t dp = fep->fce_resource;
534 
535 				i_ddi_fm_dma_err_set(dp, derr->fme_ena, status,
536 				    DDI_FM_ERR_UNEXPECTED);
537 				ddi_fm_dma_err_get(dp, derr, DDI_FME_VERSION);
538 				derr->fme_dma_handle = dp;
539 			}
540 			break;
541 		}
542 		mutex_exit(&fcp->fc_lock);
543 	}
544 	return (fatal ? DDI_FM_FATAL : nonfatal ? DDI_FM_NONFATAL :
545 	    DDI_FM_UNKNOWN);
546 }
547 
548 /*
549  * Check error state against the handle resource stored in the specified
550  * FM cache.  If tdip != NULL, we check only the cache entries for tdip.
551  * The caller must ensure that tdip is valid throughout the call and
552  * all FM data structures can be safely accesses.
553  *
554  * If tdip == NULL, we check all children that have registered their
555  * FM_DMA_CHK or FM_ACC_CHK capabilities.
556  *
557  * The following status values may be returned:
558  *
559  *	DDI_FM_FATAL - if at least one cache entry comparison yields a
560  *			fatal error.
561  *
562  *	DDI_FM_NONFATAL - if at least one cache entry comparison yields a
563  *			non-fatal error and no comparison yields a fatal error.
564  *
565  *	DDI_FM_UNKNOWN - cache entry comparisons did not yield fatal or
566  *			non-fatal errors.
567  *
568  */
569 int
570 ndi_fmc_error(dev_info_t *dip, dev_info_t *tdip, int flag, uint64_t ena,
571     const void *bus_err_state)
572 {
573 	int status, fatal = 0, nonfatal = 0;
574 	ddi_fm_error_t derr;
575 	struct i_ddi_fmhdl *fmhdl;
576 	struct i_ddi_fmtgt *tgt;
577 
578 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
579 
580 	i_ddi_fm_handler_enter(dip);
581 	fmhdl = DEVI(dip)->devi_fmhdl;
582 	ASSERT(fmhdl);
583 
584 	bzero(&derr, sizeof (ddi_fm_error_t));
585 	derr.fme_version = DDI_FME_VERSION;
586 	derr.fme_flag = DDI_FM_ERR_UNEXPECTED;
587 	derr.fme_ena = ena;
588 
589 	for (tgt = fmhdl->fh_tgts; tgt != NULL; tgt = tgt->ft_next) {
590 
591 		if (tdip != NULL && tdip != tgt->ft_dip)
592 			continue;
593 
594 		/*
595 		 * Attempt to find the entry in this childs handle cache
596 		 */
597 		status = ndi_fmc_entry_error(tgt->ft_dip, flag, &derr,
598 		    bus_err_state);
599 
600 		if (status == DDI_FM_FATAL)
601 			++fatal;
602 		else if (status == DDI_FM_NONFATAL)
603 			++nonfatal;
604 		else
605 			continue;
606 
607 		/*
608 		 * Call our child to process this error.
609 		 */
610 		status = tgt->ft_errhdl->eh_func(tgt->ft_dip, &derr,
611 		    tgt->ft_errhdl->eh_impl);
612 
613 		if (status == DDI_FM_FATAL)
614 			++fatal;
615 		else if (status == DDI_FM_NONFATAL)
616 			++nonfatal;
617 	}
618 
619 	i_ddi_fm_handler_exit(dip);
620 
621 	if (fatal)
622 		return (DDI_FM_FATAL);
623 	else if (nonfatal)
624 		return (DDI_FM_NONFATAL);
625 
626 	return (DDI_FM_UNKNOWN);
627 }
628 
629 int
630 ndi_fmc_entry_error_all(dev_info_t *dip, int flag, ddi_fm_error_t *derr)
631 {
632 	ndi_fmc_t *fcp = NULL;
633 	ndi_fmcentry_t *fep;
634 	struct i_ddi_fmhdl *fmhdl;
635 	int nonfatal = 0;
636 
637 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
638 
639 	fmhdl = DEVI(dip)->devi_fmhdl;
640 	ASSERT(fmhdl);
641 
642 	if (flag == DMA_HANDLE && DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
643 		fcp = fmhdl->fh_dma_cache;
644 		ASSERT(fcp);
645 	} else if (flag == ACC_HANDLE && DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
646 		fcp = fmhdl->fh_acc_cache;
647 		ASSERT(fcp);
648 	}
649 
650 	if (fcp != NULL) {
651 		/*
652 		 * Check active resource entries
653 		 */
654 		mutex_enter(&fcp->fc_lock);
655 		for (fep = fcp->fc_active->fce_next; fep != NULL;
656 		    fep = fep->fce_next) {
657 			/* Set the error for this resource handle */
658 			nonfatal++;
659 			if (flag == ACC_HANDLE) {
660 				ddi_acc_handle_t ap = fep->fce_resource;
661 
662 				i_ddi_fm_acc_err_set(ap, derr->fme_ena,
663 				    DDI_FM_NONFATAL, DDI_FM_ERR_UNEXPECTED);
664 				ddi_fm_acc_err_get(ap, derr, DDI_FME_VERSION);
665 				derr->fme_acc_handle = ap;
666 			} else {
667 				ddi_dma_handle_t dp = fep->fce_resource;
668 
669 				i_ddi_fm_dma_err_set(dp, derr->fme_ena,
670 				    DDI_FM_NONFATAL, DDI_FM_ERR_UNEXPECTED);
671 				ddi_fm_dma_err_get(dp, derr, DDI_FME_VERSION);
672 				derr->fme_dma_handle = dp;
673 			}
674 		}
675 		mutex_exit(&fcp->fc_lock);
676 	}
677 	return (nonfatal ? DDI_FM_NONFATAL : DDI_FM_UNKNOWN);
678 }
679 
680 /*
681  * Dispatch registered error handlers for dip.  If tdip != NULL, only
682  * the error handler (if available) for tdip is invoked.  Otherwise,
683  * all registered error handlers are invoked.
684  *
685  * The following status values may be returned:
686  *
687  *	DDI_FM_FATAL - if at least one error handler returns a
688  *			fatal error.
689  *
690  *	DDI_FM_NONFATAL - if at least one error handler returns a
691  *			non-fatal error and none returned a fatal error.
692  *
693  *	DDI_FM_UNKNOWN - if at least one error handler returns
694  *			unknown status and none return fatal or non-fatal.
695  *
696  *	DDI_FM_OK - if all error handlers return DDI_FM_OK
697  */
698 int
699 ndi_fm_handler_dispatch(dev_info_t *dip, dev_info_t *tdip,
700     const ddi_fm_error_t *nerr)
701 {
702 	int status;
703 	int unknown = 0, fatal = 0, nonfatal = 0;
704 	struct i_ddi_fmhdl *hdl;
705 	struct i_ddi_fmtgt *tgt;
706 
707 	status = DDI_FM_UNKNOWN;
708 
709 	i_ddi_fm_handler_enter(dip);
710 	hdl = DEVI(dip)->devi_fmhdl;
711 	tgt = hdl->fh_tgts;
712 	while (tgt != NULL) {
713 		if (tdip == NULL || tdip == tgt->ft_dip) {
714 			struct i_ddi_errhdl *errhdl;
715 
716 			errhdl = tgt->ft_errhdl;
717 			status = errhdl->eh_func(tgt->ft_dip, nerr,
718 			    errhdl->eh_impl);
719 
720 			if (status == DDI_FM_FATAL)
721 				++fatal;
722 			else if (status == DDI_FM_NONFATAL)
723 				++nonfatal;
724 			else if (status == DDI_FM_UNKNOWN)
725 				++unknown;
726 
727 			/* Only interested in one target */
728 			if (tdip != NULL)
729 				break;
730 		}
731 		tgt = tgt->ft_next;
732 	}
733 	i_ddi_fm_handler_exit(dip);
734 
735 	if (fatal)
736 		return (DDI_FM_FATAL);
737 	else if (nonfatal)
738 		return (DDI_FM_NONFATAL);
739 	else if (unknown)
740 		return (DDI_FM_UNKNOWN);
741 	else
742 		return (DDI_FM_OK);
743 }
744 
745 /*
746  * Set error status for specified access or DMA handle
747  *
748  * May be called in any context but caller must insure validity of
749  * handle.
750  */
751 void
752 ndi_fm_acc_err_set(ddi_acc_handle_t handle, ddi_fm_error_t *dfe)
753 {
754 	i_ddi_fm_acc_err_set(handle, dfe->fme_ena, dfe->fme_status,
755 	    dfe->fme_flag);
756 }
757 
758 void
759 ndi_fm_dma_err_set(ddi_dma_handle_t handle, ddi_fm_error_t *dfe)
760 {
761 	i_ddi_fm_dma_err_set(handle, dfe->fme_ena, dfe->fme_status,
762 	    dfe->fme_flag);
763 }
764 
765 /*
766  * Call parent busop fm initialization routine.
767  *
768  * Called during driver attach(1M)
769  */
770 int
771 i_ndi_busop_fm_init(dev_info_t *dip, int tcap, ddi_iblock_cookie_t *ibc)
772 {
773 	int pcap;
774 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
775 
776 	if (dip == ddi_root_node())
777 		return (ddi_system_fmcap | DDI_FM_EREPORT_CAPABLE);
778 
779 	/* Valid operation for BUSO_REV_6 and above */
780 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
781 		return (DDI_FM_NOT_CAPABLE);
782 
783 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_init == NULL)
784 		return (DDI_FM_NOT_CAPABLE);
785 
786 	pcap = (*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_init)
787 	    (pdip, dip, tcap, ibc);
788 
789 	return (pcap);
790 }
791 
792 /*
793  * Call parent busop fm clean-up routine.
794  *
795  * Called during driver detach(1M)
796  */
797 void
798 i_ndi_busop_fm_fini(dev_info_t *dip)
799 {
800 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
801 
802 	if (dip == ddi_root_node())
803 		return;
804 
805 	/* Valid operation for BUSO_REV_6 and above */
806 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
807 		return;
808 
809 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_fini == NULL)
810 		return;
811 
812 	(*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_fini)(pdip, dip);
813 }
814 
815 /*
816  * The following routines provide exclusive access to a nexus resource
817  *
818  * These busops may be called in user or kernel driver context.
819  */
820 void
821 i_ndi_busop_access_enter(dev_info_t *dip, ddi_acc_handle_t handle)
822 {
823 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
824 
825 	/* Valid operation for BUSO_REV_6 and above */
826 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
827 		return;
828 
829 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_enter == NULL)
830 		return;
831 
832 	(*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_enter)
833 	    (pdip, handle);
834 }
835 
836 void
837 i_ndi_busop_access_exit(dev_info_t *dip, ddi_acc_handle_t handle)
838 {
839 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
840 
841 	/* Valid operation for BUSO_REV_6 and above */
842 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
843 		return;
844 
845 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_exit == NULL)
846 		return;
847 
848 	(*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_exit)(pdip, handle);
849 }
850