xref: /titanic_52/usr/src/uts/common/os/ndifm.c (revision 567c0b92c2af3154cccac71336940521945f8ae8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Fault Management for Nexus Device Drivers
28  *
29  * In addition to implementing and supporting Fault Management for Device
30  * Drivers (ddifm.c), nexus drivers must support their children by
31  * reporting FM capabilities, intializing interrupt block cookies
32  * for error handling callbacks and caching mapped resources for lookup
33  * during the detection of an IO transaction error.
34  *
35  * It is typically the nexus driver that receives an error indication
36  * for a fault that may have occurred in the data path of an IO transaction.
37  * Errors may be detected or received via an interrupt, a callback from
38  * another subsystem (e.g. a cpu trap) or examination of control data.
39  *
40  * Upon detection of an error, the nexus has a responsibility to alert
41  * its children of the error and the transaction associated with that
42  * error.  The actual implementation may vary depending upon the capabilities
43  * of the nexus, its underlying hardware and its children.  In this file,
44  * we provide support for typical nexus driver fault management tasks.
45  *
46  * Fault Management Initialization
47  *
48  *      Nexus drivers must implement two new busops, bus_fm_init() and
49  *      bus_fm_fini().  bus_fm_init() is called from a child nexus or device
50  *      driver and is expected to initialize any per-child state and return
51  *      the FM and error interrupt priority levels of the nexus driver.
52  *      Similarly, bus_fm_fini() is called by child drivers and should
53  *      clean-up any resources allocated during bus_fm_init().
54  *      These functions are called from passive kernel context, typically from
55  *      driver attach(9F) and detach(9F) entry points.
56  *
57  * Error Handler Dispatching
58  *
59  *      Nexus drivers implemented to support error handler capabilities
60  *	should invoke registered error handler callbacks for child drivers
61  *	thought to be involved in the error.
62  *	ndi_fm_handler_dispatch() is used to invoke
63  *      all error handlers and returns one of the following status
64  *      indications:
65  *
66  *      DDI_FM_OK - No errors found by any child
67  *      DDI_FM_FATAL - one or more children have detected a fatal error
68  *      DDI_FM_NONFATAL - no fatal errors, but one or more children have
69  *                            detected a non-fatal error
70  *
71  *      ndi_fm_handler_dispatch() may be called in any context
72  *      subject to the constraints specified by the interrupt iblock cookie
73  *      returned during initialization.
74  *
75  * Protected Accesses
76  *
77  *      When an access handle is mapped or a DMA handle is bound via the
78  *      standard busops, bus_map() or bus_dma_bindhdl(), a child driver
79  *      implemented to support DDI_FM_ACCCHK_CAPABLE or
80  *	DDI_FM_DMACHK_CAPABLE capabilites
81  *	expects the nexus to flag any errors detected for transactions
82  *	associated with the mapped or bound handles.
83  *
84  *      Children nexus or device drivers will set the following flags
85  *      in their ddi_device_access or dma_attr_flags when requesting
86  *      the an access or DMA handle mapping:
87  *
88  *      DDI_DMA_FLAGERR - nexus should set error status for any errors
89  *                              detected for a failed DMA transaction.
90  *      DDI_ACC_FLAGERR - nexus should set error status for any errors
91  *                              detected for a failed PIO transaction.
92  *
93  *      A nexus is expected to provide additional error detection and
94  *      handling for handles with these flags set.
95  *
96  * Exclusive Bus Access
97  *
98  *      In cases where a driver requires a high level of fault tolerance
99  *      for a programmed IO transaction, it is neccessary to grant exclusive
100  *      access to the bus resource.  Exclusivity guarantees that a fault
101  *      resulting from a transaction on the bus can be easily traced and
102  *      reported to the driver requesting the transaction.
103  *
104  *      Nexus drivers must implement two new busops to support exclusive
105  *      access, bus_fm_access_enter() and bus_fm_access_exit().  The IO
106  *      framework will use these functions when it must set-up access
107  *      handles that set devacc_attr_access to DDI_ACC_CAUTIOUS in
108  *      their ddi_device_acc_attr_t request.
109  *
110  *      Upon receipt of a bus_fm_access_enter() request, the nexus must prevent
111  *      all other access requests until it receives bus_fm_access_exit()
112  *      for the requested bus instance. bus_fm_access_enter() and
113  *	bus_fm_access_exit() may be called from user, kernel or kernel
114  *	interrupt context.
115  *
116  * Access and DMA Handle Caching
117  *
118  *      To aid a nexus driver in associating access or DMA handles with
119  *      a detected error, the nexus should cache all handles that are
120  *      associated with DDI_ACC_FLAGERR, DDI_ACC_CAUTIOUS_ACC or
121  *	DDI_DMA_FLAGERR requests from its children.  ndi_fmc_insert() is
122  *	called by a nexus to cache handles with the above protection flags
123  *	and ndi_fmc_remove() is called when that handle is unmapped or
124  *	unbound by the requesting child.  ndi_fmc_insert() and
125  *	ndi_fmc_remove() may be called from any user or kernel context.
126  *
127  *	FM cache element is implemented by kmem_cache. The elements are
128  *	stored in a doubly-linked searchable list.  When a handle is created,
129  *	ndi_fm_insert() allocates an entry from the kmem_cache and inserts
130  *	the entry to the head of the list.  When a handle is unmapped
131  *	or unbound, ndi_fm_remove() removes its associated cache entry from
132  *	the list.
133  *
134  *      Upon detection of an error, the nexus may invoke ndi_fmc_error() to
135  *      iterate over the handle cache of one or more of its FM compliant
136  *      children.  A comparison callback function is provided upon each
137  *      invocation of ndi_fmc_error() to tell the IO framework if a
138  *      handle is associated with an error.  If so, the framework will
139  *      set the error status for that handle before returning from
140  *      ndi_fmc_error().
141  *
142  *      ndi_fmc_error() may be called in any context
143  *      subject to the constraints specified by the interrupt iblock cookie
144  *      returned during initialization of the nexus and its children.
145  *
146  */
147 
148 #include <sys/types.h>
149 #include <sys/param.h>
150 #include <sys/debug.h>
151 #include <sys/sunddi.h>
152 #include <sys/sunndi.h>
153 #include <sys/ddi.h>
154 #include <sys/ndi_impldefs.h>
155 #include <sys/devctl.h>
156 #include <sys/nvpair.h>
157 #include <sys/ddifm.h>
158 #include <sys/ndifm.h>
159 #include <sys/spl.h>
160 #include <sys/sysmacros.h>
161 #include <sys/devops.h>
162 #include <sys/atomic.h>
163 #include <sys/kmem.h>
164 #include <sys/fm/io/ddi.h>
165 
166 kmem_cache_t *ndi_fm_entry_cache;
167 
168 void
169 ndi_fm_init(void)
170 {
171 	ndi_fm_entry_cache = kmem_cache_create("ndi_fm_entry_cache",
172 	    sizeof (ndi_fmcentry_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
173 }
174 
175 /*
176  * Allocate and initialize a fault management resource cache
177  * A fault management cache consists of a set of cache elements that
178  * are allocated from "ndi_fm_entry_cache".
179  */
180 /* ARGSUSED */
181 void
182 i_ndi_fmc_create(ndi_fmc_t **fcpp, int qlen, ddi_iblock_cookie_t ibc)
183 {
184 	ndi_fmc_t *fcp;
185 
186 	fcp = kmem_zalloc(sizeof (ndi_fmc_t), KM_SLEEP);
187 	mutex_init(&fcp->fc_lock, NULL, MUTEX_DRIVER, ibc);
188 
189 	*fcpp = fcp;
190 }
191 
192 /*
193  * Destroy and resources associated with the given fault management cache.
194  */
195 void
196 i_ndi_fmc_destroy(ndi_fmc_t *fcp)
197 {
198 	ndi_fmcentry_t *fep, *pp;
199 
200 	if (fcp == NULL)
201 		return;
202 
203 	/* Free all the cached entries, this should not happen though */
204 	mutex_enter(&fcp->fc_lock);
205 	for (fep = fcp->fc_head; fep != NULL; fep = pp) {
206 		pp = fep->fce_next;
207 		kmem_cache_free(ndi_fm_entry_cache, fep);
208 	}
209 	mutex_exit(&fcp->fc_lock);
210 	mutex_destroy(&fcp->fc_lock);
211 	kmem_free(fcp, sizeof (ndi_fmc_t));
212 }
213 
214 /*
215  * ndi_fmc_insert -
216  * 	Add a new entry to the specified cache.
217  *
218  * 	This function must be called at or below LOCK_LEVEL
219  */
220 void
221 ndi_fmc_insert(dev_info_t *dip, int flag, void *resource, void *bus_specific)
222 {
223 	struct dev_info *devi = DEVI(dip);
224 	ndi_fmc_t *fcp;
225 	ndi_fmcentry_t *fep, **fpp;
226 	struct i_ddi_fmhdl *fmhdl;
227 
228 	ASSERT(devi);
229 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
230 
231 	fmhdl = devi->devi_fmhdl;
232 	if (fmhdl == NULL) {
233 		return;
234 	}
235 
236 	if (flag == DMA_HANDLE) {
237 		if (!DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
238 			return;
239 		}
240 		fcp = fmhdl->fh_dma_cache;
241 		fpp = &((ddi_dma_impl_t *)resource)->dmai_error.err_fep;
242 	} else if (flag == ACC_HANDLE) {
243 		if (!DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
244 			i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL,
245 			    DDI_NOSLEEP);
246 			return;
247 		}
248 		fcp = fmhdl->fh_acc_cache;
249 		fpp = &((ddi_acc_impl_t *)resource)->ahi_err->err_fep;
250 	}
251 
252 	fep = kmem_cache_alloc(ndi_fm_entry_cache, KM_NOSLEEP);
253 	if (fep == NULL) {
254 		atomic_inc_64(&fmhdl->fh_kstat.fek_fmc_full.value.ui64);
255 		return;
256 	}
257 
258 	/*
259 	 * Set-up the handle resource and bus_specific information.
260 	 * Also remember the pointer back to the cache for quick removal.
261 	 */
262 	fep->fce_bus_specific = bus_specific;
263 	fep->fce_resource = resource;
264 	fep->fce_next = NULL;
265 
266 	/* Add entry to the end of the active list */
267 	mutex_enter(&fcp->fc_lock);
268 	ASSERT(*fpp == NULL);
269 	*fpp = fep;
270 	fep->fce_prev = fcp->fc_tail;
271 	if (fcp->fc_tail != NULL)
272 		fcp->fc_tail->fce_next = fep;
273 	else
274 		fcp->fc_head = fep;
275 	fcp->fc_tail = fep;
276 	mutex_exit(&fcp->fc_lock);
277 }
278 
279 /*
280  * 	Remove an entry from the specified cache of access or dma mappings
281  *
282  * 	This function must be called at or below LOCK_LEVEL.
283  */
284 void
285 ndi_fmc_remove(dev_info_t *dip, int flag, const void *resource)
286 {
287 	ndi_fmc_t *fcp;
288 	ndi_fmcentry_t *fep;
289 	struct dev_info *devi = DEVI(dip);
290 	struct i_ddi_fmhdl *fmhdl;
291 
292 	ASSERT(devi);
293 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
294 
295 	fmhdl = devi->devi_fmhdl;
296 	if (fmhdl == NULL) {
297 		return;
298 	}
299 
300 	/* Find cache entry pointer for this resource */
301 	if (flag == DMA_HANDLE) {
302 		if (!DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
303 			return;
304 		}
305 		fcp = fmhdl->fh_dma_cache;
306 
307 		ASSERT(fcp);
308 
309 		mutex_enter(&fcp->fc_lock);
310 		fep = ((ddi_dma_impl_t *)resource)->dmai_error.err_fep;
311 		((ddi_dma_impl_t *)resource)->dmai_error.err_fep = NULL;
312 	} else if (flag == ACC_HANDLE) {
313 		if (!DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
314 			i_ddi_drv_ereport_post(dip, DVR_EFMCAP, NULL,
315 			    DDI_NOSLEEP);
316 			return;
317 		}
318 		fcp = fmhdl->fh_acc_cache;
319 
320 		ASSERT(fcp);
321 
322 		mutex_enter(&fcp->fc_lock);
323 		fep = ((ddi_acc_impl_t *)resource)->ahi_err->err_fep;
324 		((ddi_acc_impl_t *)resource)->ahi_err->err_fep = NULL;
325 	} else {
326 		return;
327 	}
328 
329 	/*
330 	 * Resource not in cache, return
331 	 */
332 	if (fep == NULL) {
333 		mutex_exit(&fcp->fc_lock);
334 		atomic_inc_64(&fmhdl->fh_kstat.fek_fmc_miss.value.ui64);
335 		return;
336 	}
337 
338 	/*
339 	 * Updates to FM cache pointers require us to grab fmc_lock
340 	 * to synchronize access to the cache for ndi_fmc_insert()
341 	 * and ndi_fmc_error()
342 	 */
343 	if (fep == fcp->fc_head)
344 		fcp->fc_head = fep->fce_next;
345 	else
346 		fep->fce_prev->fce_next = fep->fce_next;
347 	if (fep == fcp->fc_tail)
348 		fcp->fc_tail = fep->fce_prev;
349 	else
350 		fep->fce_next->fce_prev = fep->fce_prev;
351 	mutex_exit(&fcp->fc_lock);
352 
353 	kmem_cache_free(ndi_fm_entry_cache, fep);
354 }
355 
356 int
357 ndi_fmc_entry_error(dev_info_t *dip, int flag, ddi_fm_error_t *derr,
358     const void *bus_err_state)
359 {
360 	int status, fatal = 0, nonfatal = 0;
361 	ndi_fmc_t *fcp = NULL;
362 	ndi_fmcentry_t *fep;
363 	struct i_ddi_fmhdl *fmhdl;
364 
365 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
366 
367 	fmhdl = DEVI(dip)->devi_fmhdl;
368 	ASSERT(fmhdl);
369 	status = DDI_FM_UNKNOWN;
370 
371 	if (flag == DMA_HANDLE && DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
372 		fcp = fmhdl->fh_dma_cache;
373 		ASSERT(fcp);
374 	} else if (flag == ACC_HANDLE && DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
375 		fcp = fmhdl->fh_acc_cache;
376 		ASSERT(fcp);
377 	}
378 
379 	if (fcp != NULL) {
380 
381 		/*
382 		 * Check active resource entries
383 		 */
384 		mutex_enter(&fcp->fc_lock);
385 		for (fep = fcp->fc_head; fep != NULL; fep = fep->fce_next) {
386 			ddi_fmcompare_t compare_func;
387 
388 			/*
389 			 * Compare captured error state with handle
390 			 * resources.  During the comparison and
391 			 * subsequent error handling, we block
392 			 * attempts to free the cache entry.
393 			 */
394 			compare_func = (flag == ACC_HANDLE) ?
395 			    i_ddi_fm_acc_err_cf_get((ddi_acc_handle_t)
396 			    fep->fce_resource) :
397 			    i_ddi_fm_dma_err_cf_get((ddi_dma_handle_t)
398 			    fep->fce_resource);
399 
400 			if (compare_func == NULL) /* unbound or not FLAGERR */
401 				continue;
402 
403 			status = compare_func(dip, fep->fce_resource,
404 			    bus_err_state, fep->fce_bus_specific);
405 			if (status == DDI_FM_UNKNOWN || status == DDI_FM_OK)
406 				continue;
407 
408 			if (status == DDI_FM_FATAL)
409 				++fatal;
410 			else if (status == DDI_FM_NONFATAL)
411 				++nonfatal;
412 
413 			/* Set the error for this resource handle */
414 			if (flag == ACC_HANDLE) {
415 				ddi_acc_handle_t ap = fep->fce_resource;
416 
417 				i_ddi_fm_acc_err_set(ap, derr->fme_ena, status,
418 				    DDI_FM_ERR_UNEXPECTED);
419 				ddi_fm_acc_err_get(ap, derr, DDI_FME_VERSION);
420 				derr->fme_acc_handle = ap;
421 			} else {
422 				ddi_dma_handle_t dp = fep->fce_resource;
423 
424 				i_ddi_fm_dma_err_set(dp, derr->fme_ena, status,
425 				    DDI_FM_ERR_UNEXPECTED);
426 				ddi_fm_dma_err_get(dp, derr, DDI_FME_VERSION);
427 				derr->fme_dma_handle = dp;
428 			}
429 		}
430 		mutex_exit(&fcp->fc_lock);
431 	}
432 	return (fatal ? DDI_FM_FATAL : nonfatal ? DDI_FM_NONFATAL :
433 	    DDI_FM_UNKNOWN);
434 }
435 
436 /*
437  * Check error state against the handle resource stored in the specified
438  * FM cache.  If tdip != NULL, we check only the cache entries for tdip.
439  * The caller must ensure that tdip is valid throughout the call and
440  * all FM data structures can be safely accesses.
441  *
442  * If tdip == NULL, we check all children that have registered their
443  * FM_DMA_CHK or FM_ACC_CHK capabilities.
444  *
445  * The following status values may be returned:
446  *
447  *	DDI_FM_FATAL - if at least one cache entry comparison yields a
448  *			fatal error.
449  *
450  *	DDI_FM_NONFATAL - if at least one cache entry comparison yields a
451  *			non-fatal error and no comparison yields a fatal error.
452  *
453  *	DDI_FM_UNKNOWN - cache entry comparisons did not yield fatal or
454  *			non-fatal errors.
455  *
456  */
457 int
458 ndi_fmc_error(dev_info_t *dip, dev_info_t *tdip, int flag, uint64_t ena,
459     const void *bus_err_state)
460 {
461 	int status, fatal = 0, nonfatal = 0;
462 	ddi_fm_error_t derr;
463 	struct i_ddi_fmhdl *fmhdl;
464 	struct i_ddi_fmtgt *tgt;
465 
466 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
467 
468 	i_ddi_fm_handler_enter(dip);
469 	fmhdl = DEVI(dip)->devi_fmhdl;
470 	ASSERT(fmhdl);
471 
472 	bzero(&derr, sizeof (ddi_fm_error_t));
473 	derr.fme_version = DDI_FME_VERSION;
474 	derr.fme_flag = DDI_FM_ERR_UNEXPECTED;
475 	derr.fme_ena = ena;
476 
477 	for (tgt = fmhdl->fh_tgts; tgt != NULL; tgt = tgt->ft_next) {
478 
479 		if (tdip != NULL && tdip != tgt->ft_dip)
480 			continue;
481 
482 		/*
483 		 * Attempt to find the entry in this childs handle cache
484 		 */
485 		status = ndi_fmc_entry_error(tgt->ft_dip, flag, &derr,
486 		    bus_err_state);
487 
488 		if (status == DDI_FM_FATAL)
489 			++fatal;
490 		else if (status == DDI_FM_NONFATAL)
491 			++nonfatal;
492 		else
493 			continue;
494 
495 		/*
496 		 * Call our child to process this error.
497 		 */
498 		status = tgt->ft_errhdl->eh_func(tgt->ft_dip, &derr,
499 		    tgt->ft_errhdl->eh_impl);
500 
501 		if (status == DDI_FM_FATAL)
502 			++fatal;
503 		else if (status == DDI_FM_NONFATAL)
504 			++nonfatal;
505 	}
506 
507 	i_ddi_fm_handler_exit(dip);
508 
509 	if (fatal)
510 		return (DDI_FM_FATAL);
511 	else if (nonfatal)
512 		return (DDI_FM_NONFATAL);
513 
514 	return (DDI_FM_UNKNOWN);
515 }
516 
517 int
518 ndi_fmc_entry_error_all(dev_info_t *dip, int flag, ddi_fm_error_t *derr)
519 {
520 	ndi_fmc_t *fcp = NULL;
521 	ndi_fmcentry_t *fep;
522 	struct i_ddi_fmhdl *fmhdl;
523 	int nonfatal = 0;
524 
525 	ASSERT(flag == DMA_HANDLE || flag == ACC_HANDLE);
526 
527 	fmhdl = DEVI(dip)->devi_fmhdl;
528 	ASSERT(fmhdl);
529 
530 	if (flag == DMA_HANDLE && DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap)) {
531 		fcp = fmhdl->fh_dma_cache;
532 		ASSERT(fcp);
533 	} else if (flag == ACC_HANDLE && DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap)) {
534 		fcp = fmhdl->fh_acc_cache;
535 		ASSERT(fcp);
536 	}
537 
538 	if (fcp != NULL) {
539 		/*
540 		 * Check active resource entries
541 		 */
542 		mutex_enter(&fcp->fc_lock);
543 		for (fep = fcp->fc_head; fep != NULL; fep = fep->fce_next) {
544 			ddi_fmcompare_t compare_func;
545 
546 			compare_func = (flag == ACC_HANDLE) ?
547 			    i_ddi_fm_acc_err_cf_get((ddi_acc_handle_t)
548 			    fep->fce_resource) :
549 			    i_ddi_fm_dma_err_cf_get((ddi_dma_handle_t)
550 			    fep->fce_resource);
551 
552 			if (compare_func == NULL) /* unbound or not FLAGERR */
553 				continue;
554 
555 			/* Set the error for this resource handle */
556 			nonfatal++;
557 
558 			if (flag == ACC_HANDLE) {
559 				ddi_acc_handle_t ap = fep->fce_resource;
560 
561 				i_ddi_fm_acc_err_set(ap, derr->fme_ena,
562 				    DDI_FM_NONFATAL, DDI_FM_ERR_UNEXPECTED);
563 				ddi_fm_acc_err_get(ap, derr, DDI_FME_VERSION);
564 				derr->fme_acc_handle = ap;
565 			} else {
566 				ddi_dma_handle_t dp = fep->fce_resource;
567 
568 				i_ddi_fm_dma_err_set(dp, derr->fme_ena,
569 				    DDI_FM_NONFATAL, DDI_FM_ERR_UNEXPECTED);
570 				ddi_fm_dma_err_get(dp, derr, DDI_FME_VERSION);
571 				derr->fme_dma_handle = dp;
572 			}
573 		}
574 		mutex_exit(&fcp->fc_lock);
575 	}
576 	return (nonfatal ? DDI_FM_NONFATAL : DDI_FM_UNKNOWN);
577 }
578 
579 /*
580  * Dispatch registered error handlers for dip.  If tdip != NULL, only
581  * the error handler (if available) for tdip is invoked.  Otherwise,
582  * all registered error handlers are invoked.
583  *
584  * The following status values may be returned:
585  *
586  *	DDI_FM_FATAL - if at least one error handler returns a
587  *			fatal error.
588  *
589  *	DDI_FM_NONFATAL - if at least one error handler returns a
590  *			non-fatal error and none returned a fatal error.
591  *
592  *	DDI_FM_UNKNOWN - if at least one error handler returns
593  *			unknown status and none return fatal or non-fatal.
594  *
595  *	DDI_FM_OK - if all error handlers return DDI_FM_OK
596  */
597 int
598 ndi_fm_handler_dispatch(dev_info_t *dip, dev_info_t *tdip,
599     const ddi_fm_error_t *nerr)
600 {
601 	int status;
602 	int unknown = 0, fatal = 0, nonfatal = 0;
603 	struct i_ddi_fmhdl *hdl;
604 	struct i_ddi_fmtgt *tgt;
605 
606 	status = DDI_FM_UNKNOWN;
607 
608 	i_ddi_fm_handler_enter(dip);
609 	hdl = DEVI(dip)->devi_fmhdl;
610 	tgt = hdl->fh_tgts;
611 	while (tgt != NULL) {
612 		if (tdip == NULL || tdip == tgt->ft_dip) {
613 			struct i_ddi_errhdl *errhdl;
614 
615 			errhdl = tgt->ft_errhdl;
616 			status = errhdl->eh_func(tgt->ft_dip, nerr,
617 			    errhdl->eh_impl);
618 
619 			if (status == DDI_FM_FATAL)
620 				++fatal;
621 			else if (status == DDI_FM_NONFATAL)
622 				++nonfatal;
623 			else if (status == DDI_FM_UNKNOWN)
624 				++unknown;
625 
626 			/* Only interested in one target */
627 			if (tdip != NULL)
628 				break;
629 		}
630 		tgt = tgt->ft_next;
631 	}
632 	i_ddi_fm_handler_exit(dip);
633 
634 	if (fatal)
635 		return (DDI_FM_FATAL);
636 	else if (nonfatal)
637 		return (DDI_FM_NONFATAL);
638 	else if (unknown)
639 		return (DDI_FM_UNKNOWN);
640 	else
641 		return (DDI_FM_OK);
642 }
643 
644 /*
645  * Set error status for specified access or DMA handle
646  *
647  * May be called in any context but caller must insure validity of
648  * handle.
649  */
650 void
651 ndi_fm_acc_err_set(ddi_acc_handle_t handle, ddi_fm_error_t *dfe)
652 {
653 	i_ddi_fm_acc_err_set(handle, dfe->fme_ena, dfe->fme_status,
654 	    dfe->fme_flag);
655 }
656 
657 void
658 ndi_fm_dma_err_set(ddi_dma_handle_t handle, ddi_fm_error_t *dfe)
659 {
660 	i_ddi_fm_dma_err_set(handle, dfe->fme_ena, dfe->fme_status,
661 	    dfe->fme_flag);
662 }
663 
664 /*
665  * Call parent busop fm initialization routine.
666  *
667  * Called during driver attach(1M)
668  */
669 int
670 i_ndi_busop_fm_init(dev_info_t *dip, int tcap, ddi_iblock_cookie_t *ibc)
671 {
672 	int pcap;
673 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
674 
675 	if (dip == ddi_root_node())
676 		return (ddi_system_fmcap | DDI_FM_EREPORT_CAPABLE);
677 
678 	/* Valid operation for BUSO_REV_6 and above */
679 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
680 		return (DDI_FM_NOT_CAPABLE);
681 
682 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_init == NULL)
683 		return (DDI_FM_NOT_CAPABLE);
684 
685 	pcap = (*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_init)
686 	    (pdip, dip, tcap, ibc);
687 
688 	return (pcap);
689 }
690 
691 /*
692  * Call parent busop fm clean-up routine.
693  *
694  * Called during driver detach(1M)
695  */
696 void
697 i_ndi_busop_fm_fini(dev_info_t *dip)
698 {
699 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
700 
701 	if (dip == ddi_root_node())
702 		return;
703 
704 	/* Valid operation for BUSO_REV_6 and above */
705 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
706 		return;
707 
708 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_fini == NULL)
709 		return;
710 
711 	(*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_fini)(pdip, dip);
712 }
713 
714 /*
715  * The following routines provide exclusive access to a nexus resource
716  *
717  * These busops may be called in user or kernel driver context.
718  */
719 void
720 i_ndi_busop_access_enter(dev_info_t *dip, ddi_acc_handle_t handle)
721 {
722 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
723 
724 	/* Valid operation for BUSO_REV_6 and above */
725 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
726 		return;
727 
728 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_enter == NULL)
729 		return;
730 
731 	(*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_enter)
732 	    (pdip, handle);
733 }
734 
735 void
736 i_ndi_busop_access_exit(dev_info_t *dip, ddi_acc_handle_t handle)
737 {
738 	dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
739 
740 	/* Valid operation for BUSO_REV_6 and above */
741 	if (DEVI(pdip)->devi_ops->devo_bus_ops->busops_rev < BUSO_REV_6)
742 		return;
743 
744 	if (DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_exit == NULL)
745 		return;
746 
747 	(*DEVI(pdip)->devi_ops->devo_bus_ops->bus_fm_access_exit)(pdip, handle);
748 }
749