xref: /illumos-gate/usr/src/uts/common/io/dcopy.c (revision b31b5de1357c915fe7dab4d9646d9d84f9fe69bc)
1  /*
2   * CDDL HEADER START
3   *
4   * The contents of this file are subject to the terms of the
5   * Common Development and Distribution License (the "License").
6   * You may not use this file except in compliance with the License.
7   *
8   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9   * or http://www.opensolaris.org/os/licensing.
10   * See the License for the specific language governing permissions
11   * and limitations under the License.
12   *
13   * When distributing Covered Code, include this CDDL HEADER in each
14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15   * If applicable, add the following below this CDDL HEADER, with the
16   * fields enclosed by brackets "[]" replaced with your own identifying
17   * information: Portions Copyright [yyyy] [name of copyright owner]
18   *
19   * CDDL HEADER END
20   */
21  
22  /*
23   * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24   * Use is subject to license terms.
25   */
26  
27  #pragma ident	"%Z%%M%	%I%	%E% SMI"
28  
29  /*
30   * dcopy.c
31   *    dcopy misc module
32   */
33  
34  #include <sys/conf.h>
35  #include <sys/kmem.h>
36  #include <sys/ddi.h>
37  #include <sys/sunddi.h>
38  #include <sys/modctl.h>
39  #include <sys/sysmacros.h>
40  #include <sys/atomic.h>
41  
42  
43  #include <sys/dcopy.h>
44  #include <sys/dcopy_device.h>
45  
46  
47  /* Number of entries per channel to allocate */
48  uint_t dcopy_channel_size = 1024;
49  
50  
51  typedef struct dcopy_list_s {
52  	list_t			dl_list;
53  	kmutex_t		dl_mutex;
54  	uint_t			dl_cnt; /* num entries on list */
55  } dcopy_list_t;
56  
57  /* device state for register/unregister */
58  struct dcopy_device_s {
59  	/* DMA device drivers private pointer */
60  	void			*dc_device_private;
61  
62  	/* to track list of channels from this DMA device */
63  	dcopy_list_t		dc_devchan_list;
64  	list_node_t		dc_device_list_node;
65  
66  	/*
67  	 * dc_removing_cnt track how many channels still have to be freed up
68  	 * before it's safe to allow the DMA device driver to detach.
69  	 */
70  	uint_t			dc_removing_cnt;
71  	dcopy_device_cb_t	*dc_cb;
72  
73  	dcopy_device_info_t	dc_info;
74  
75  };
76  
77  typedef struct dcopy_stats_s {
78  	kstat_named_t	cs_bytes_xfer;
79  	kstat_named_t	cs_cmd_alloc;
80  	kstat_named_t	cs_cmd_post;
81  	kstat_named_t	cs_cmd_poll;
82  	kstat_named_t	cs_notify_poll;
83  	kstat_named_t	cs_notify_pending;
84  	kstat_named_t	cs_id;
85  	kstat_named_t	cs_capabilities;
86  } dcopy_stats_t;
87  
88  /* DMA channel state */
89  struct dcopy_channel_s {
90  	/* DMA driver channel private pointer */
91  	void			*ch_channel_private;
92  
93  	/* shortcut to device callbacks */
94  	dcopy_device_cb_t	*ch_cb;
95  
96  	/*
97  	 * number of outstanding allocs for this channel. used to track when
98  	 * it's safe to free up this channel so the DMA device driver can
99  	 * detach.
100  	 */
101  	uint64_t		ch_ref_cnt;
102  
103  	/* state for if channel needs to be removed when ch_ref_cnt gets to 0 */
104  	boolean_t		ch_removing;
105  
106  	list_node_t		ch_devchan_list_node;
107  	list_node_t		ch_globalchan_list_node;
108  
109  	/*
110  	 * per channel list of commands actively blocking waiting for
111  	 * completion.
112  	 */
113  	dcopy_list_t		ch_poll_list;
114  
115  	/* pointer back to our device */
116  	struct dcopy_device_s	*ch_device;
117  
118  	dcopy_query_channel_t	ch_info;
119  
120  	kstat_t			*ch_kstat;
121  	dcopy_stats_t		ch_stat;
122  };
123  
124  /*
125   * If grabbing both device_list mutex & globalchan_list mutex,
126   * Always grab globalchan_list mutex before device_list mutex
127   */
128  typedef struct dcopy_state_s {
129  	dcopy_list_t		d_device_list;
130  	dcopy_list_t		d_globalchan_list;
131  } dcopy_state_t;
132  dcopy_state_t *dcopy_statep;
133  
134  
135  /* Module Driver Info */
136  static struct modlmisc dcopy_modlmisc = {
137  	&mod_miscops,
138  	"dcopy kernel module"
139  };
140  
141  /* Module Linkage */
142  static struct modlinkage dcopy_modlinkage = {
143  	MODREV_1,
144  	&dcopy_modlmisc,
145  	NULL
146  };
147  
148  static int dcopy_init();
149  static void dcopy_fini();
150  
151  static int dcopy_list_init(dcopy_list_t *list, size_t node_size,
152      offset_t link_offset);
153  static void dcopy_list_fini(dcopy_list_t *list);
154  static void dcopy_list_push(dcopy_list_t *list, void *list_node);
155  static void *dcopy_list_pop(dcopy_list_t *list);
156  
157  static void dcopy_device_cleanup(dcopy_device_handle_t device,
158      boolean_t do_callback);
159  
160  static int dcopy_stats_init(dcopy_handle_t channel);
161  static void dcopy_stats_fini(dcopy_handle_t channel);
162  
163  
164  /*
165   * _init()
166   */
167  int
168  _init()
169  {
170  	int e;
171  
172  	e = dcopy_init();
173  	if (e != 0) {
174  		return (e);
175  	}
176  
177  	return (mod_install(&dcopy_modlinkage));
178  }
179  
180  
181  /*
182   * _info()
183   */
184  int
185  _info(struct modinfo *modinfop)
186  {
187  	return (mod_info(&dcopy_modlinkage, modinfop));
188  }
189  
190  
191  /*
192   * _fini()
193   */
194  int
195  _fini()
196  {
197  	int e;
198  
199  	e = mod_remove(&dcopy_modlinkage);
200  	if (e != 0) {
201  		return (e);
202  	}
203  
204  	dcopy_fini();
205  
206  	return (e);
207  }
208  
209  /*
210   * dcopy_init()
211   */
212  static int
213  dcopy_init()
214  {
215  	int e;
216  
217  
218  	dcopy_statep = kmem_zalloc(sizeof (*dcopy_statep), KM_SLEEP);
219  
220  	/* Initialize the list we use to track device register/unregister */
221  	e = dcopy_list_init(&dcopy_statep->d_device_list,
222  	    sizeof (struct dcopy_device_s),
223  	    offsetof(struct dcopy_device_s, dc_device_list_node));
224  	if (e != DCOPY_SUCCESS) {
225  		goto dcopyinitfail_device;
226  	}
227  
228  	/* Initialize the list we use to track all DMA channels */
229  	e = dcopy_list_init(&dcopy_statep->d_globalchan_list,
230  	    sizeof (struct dcopy_channel_s),
231  	    offsetof(struct dcopy_channel_s, ch_globalchan_list_node));
232  	if (e != DCOPY_SUCCESS) {
233  		goto dcopyinitfail_global;
234  	}
235  
236  	return (0);
237  
238  dcopyinitfail_cback:
239  	dcopy_list_fini(&dcopy_statep->d_globalchan_list);
240  dcopyinitfail_global:
241  	dcopy_list_fini(&dcopy_statep->d_device_list);
242  dcopyinitfail_device:
243  	kmem_free(dcopy_statep, sizeof (*dcopy_statep));
244  
245  	return (-1);
246  }
247  
248  
249  /*
250   * dcopy_fini()
251   */
252  static void
253  dcopy_fini()
254  {
255  	/*
256  	 * if mod_remove was successfull, we shouldn't have any
257  	 * devices/channels to worry about.
258  	 */
259  	ASSERT(list_head(&dcopy_statep->d_globalchan_list.dl_list) == NULL);
260  	ASSERT(list_head(&dcopy_statep->d_device_list.dl_list) == NULL);
261  
262  	dcopy_list_fini(&dcopy_statep->d_globalchan_list);
263  	dcopy_list_fini(&dcopy_statep->d_device_list);
264  	kmem_free(dcopy_statep, sizeof (*dcopy_statep));
265  }
266  
267  
268  /* *** EXTERNAL INTERFACE *** */
269  /*
270   * dcopy_query()
271   */
272  void
273  dcopy_query(dcopy_query_t *query)
274  {
275  	query->dq_version = DCOPY_QUERY_V0;
276  	query->dq_num_channels = dcopy_statep->d_globalchan_list.dl_cnt;
277  }
278  
279  
280  /*
281   * dcopy_alloc()
282   */
283  /*ARGSUSED*/
284  int
285  dcopy_alloc(int flags, dcopy_handle_t *handle)
286  {
287  	dcopy_handle_t channel;
288  	dcopy_list_t *list;
289  
290  
291  	/*
292  	 * we don't use the dcopy_list_* code here because we need to due
293  	 * some non-standard stuff.
294  	 */
295  
296  	list = &dcopy_statep->d_globalchan_list;
297  
298  	/*
299  	 * if nothing is on the channel list, return DCOPY_NORESOURCES. This
300  	 * can happen if there aren't any DMA device registered.
301  	 */
302  	mutex_enter(&list->dl_mutex);
303  	channel = list_head(&list->dl_list);
304  	if (channel == NULL) {
305  		mutex_exit(&list->dl_mutex);
306  		return (DCOPY_NORESOURCES);
307  	}
308  
309  	/*
310  	 * increment the reference count, and pop the channel off the head and
311  	 * push it on the tail. This ensures we rotate through the channels.
312  	 * DMA channels are shared.
313  	 */
314  	channel->ch_ref_cnt++;
315  	list_remove(&list->dl_list, channel);
316  	list_insert_tail(&list->dl_list, channel);
317  	mutex_exit(&list->dl_mutex);
318  
319  	*handle = (dcopy_handle_t)channel;
320  	return (DCOPY_SUCCESS);
321  }
322  
323  
324  /*
325   * dcopy_free()
326   */
327  void
328  dcopy_free(dcopy_handle_t *channel)
329  {
330  	dcopy_device_handle_t device;
331  	dcopy_list_t *list;
332  	boolean_t cleanup;
333  
334  
335  	ASSERT(*channel != NULL);
336  
337  	/*
338  	 * we don't need to add the channel back to the list since we never
339  	 * removed it. decrement the reference count.
340  	 */
341  	list = &dcopy_statep->d_globalchan_list;
342  	mutex_enter(&list->dl_mutex);
343  	(*channel)->ch_ref_cnt--;
344  
345  	/*
346  	 * if we need to remove this channel, and the reference count is down
347  	 * to 0, decrement the number of channels which still need to be
348  	 * removed on the device.
349  	 */
350  	if ((*channel)->ch_removing && ((*channel)->ch_ref_cnt == 0)) {
351  		cleanup = B_FALSE;
352  		device = (*channel)->ch_device;
353  		mutex_enter(&device->dc_devchan_list.dl_mutex);
354  		device->dc_removing_cnt--;
355  		if (device->dc_removing_cnt == 0) {
356  			cleanup = B_TRUE;
357  		}
358  		mutex_exit(&device->dc_devchan_list.dl_mutex);
359  	}
360  	mutex_exit(&list->dl_mutex);
361  
362  	/*
363  	 * if there are no channels which still need to be removed, cleanup the
364  	 * device state and call back into the DMA device driver to tell them
365  	 * the device is free.
366  	 */
367  	if (cleanup) {
368  		dcopy_device_cleanup(device, B_TRUE);
369  	}
370  
371  	*channel = NULL;
372  }
373  
374  
375  /*
376   * dcopy_query_channel()
377   */
378  void
379  dcopy_query_channel(dcopy_handle_t channel, dcopy_query_channel_t *query)
380  {
381  	*query = channel->ch_info;
382  }
383  
384  
385  /*
386   * dcopy_cmd_alloc()
387   */
388  int
389  dcopy_cmd_alloc(dcopy_handle_t handle, int flags, dcopy_cmd_t *cmd)
390  {
391  	dcopy_handle_t channel;
392  	dcopy_cmd_priv_t priv;
393  	int e;
394  
395  
396  	channel = handle;
397  
398  	atomic_inc_64(&channel->ch_stat.cs_cmd_alloc.value.ui64);
399  	e = channel->ch_cb->cb_cmd_alloc(channel->ch_channel_private, flags,
400  	    cmd);
401  	if (e == DCOPY_SUCCESS) {
402  		priv = (*cmd)->dp_private;
403  		priv->pr_channel = channel;
404  		/*
405  		 * we won't initialize the blocking state until we actually
406  		 * need to block.
407  		 */
408  		priv->pr_block_init = B_FALSE;
409  	}
410  
411  	return (e);
412  }
413  
414  
415  /*
416   * dcopy_cmd_free()
417   */
418  void
419  dcopy_cmd_free(dcopy_cmd_t *cmd)
420  {
421  	dcopy_handle_t channel;
422  	dcopy_cmd_priv_t priv;
423  
424  
425  	ASSERT(*cmd != NULL);
426  
427  	priv = (*cmd)->dp_private;
428  	channel = priv->pr_channel;
429  
430  	/* if we initialized the blocking state, clean it up too */
431  	if (priv->pr_block_init) {
432  		cv_destroy(&priv->pr_cv);
433  		mutex_destroy(&priv->pr_mutex);
434  	}
435  
436  	channel->ch_cb->cb_cmd_free(channel->ch_channel_private, cmd);
437  }
438  
439  
440  /*
441   * dcopy_cmd_post()
442   */
443  int
444  dcopy_cmd_post(dcopy_cmd_t cmd)
445  {
446  	dcopy_handle_t channel;
447  	int e;
448  
449  
450  	channel = cmd->dp_private->pr_channel;
451  
452  	atomic_inc_64(&channel->ch_stat.cs_cmd_post.value.ui64);
453  	if (cmd->dp_cmd == DCOPY_CMD_COPY) {
454  		atomic_add_64(&channel->ch_stat.cs_bytes_xfer.value.ui64,
455  		    cmd->dp.copy.cc_size);
456  	}
457  	e = channel->ch_cb->cb_cmd_post(channel->ch_channel_private, cmd);
458  	if (e != DCOPY_SUCCESS) {
459  		return (e);
460  	}
461  
462  	return (DCOPY_SUCCESS);
463  }
464  
465  
466  /*
467   * dcopy_cmd_poll()
468   */
469  int
470  dcopy_cmd_poll(dcopy_cmd_t cmd, int flags)
471  {
472  	dcopy_handle_t channel;
473  	dcopy_cmd_priv_t priv;
474  	int e;
475  
476  
477  	priv = cmd->dp_private;
478  	channel = priv->pr_channel;
479  
480  	/*
481  	 * if the caller is trying to block, they needed to post the
482  	 * command with DCOPY_CMD_INTR set.
483  	 */
484  	if ((flags & DCOPY_POLL_BLOCK) && !(cmd->dp_flags & DCOPY_CMD_INTR)) {
485  		return (DCOPY_FAILURE);
486  	}
487  
488  	atomic_inc_64(&channel->ch_stat.cs_cmd_poll.value.ui64);
489  
490  repoll:
491  	e = channel->ch_cb->cb_cmd_poll(channel->ch_channel_private, cmd);
492  	if (e == DCOPY_PENDING) {
493  		/*
494  		 * if the command is still active, and the blocking flag
495  		 * is set.
496  		 */
497  		if (flags & DCOPY_POLL_BLOCK) {
498  
499  			/*
500  			 * if we haven't initialized the state, do it now. A
501  			 * command can be re-used, so it's possible it's
502  			 * already been initialized.
503  			 */
504  			if (!priv->pr_block_init) {
505  				priv->pr_block_init = B_TRUE;
506  				mutex_init(&priv->pr_mutex, NULL, MUTEX_DRIVER,
507  				    NULL);
508  				cv_init(&priv->pr_cv, NULL, CV_DRIVER, NULL);
509  				priv->pr_cmd = cmd;
510  			}
511  
512  			/* push it on the list for blocking commands */
513  			priv->pr_wait = B_TRUE;
514  			dcopy_list_push(&channel->ch_poll_list, priv);
515  
516  			mutex_enter(&priv->pr_mutex);
517  			/*
518  			 * it's possible we already cleared pr_wait before we
519  			 * grabbed the mutex.
520  			 */
521  			if (priv->pr_wait) {
522  				cv_wait(&priv->pr_cv, &priv->pr_mutex);
523  			}
524  			mutex_exit(&priv->pr_mutex);
525  
526  			/*
527  			 * the command has completed, go back and poll so we
528  			 * get the status.
529  			 */
530  			goto repoll;
531  		}
532  	}
533  
534  	return (e);
535  }
536  
537  /* *** END OF EXTERNAL INTERFACE *** */
538  
539  /*
540   * dcopy_list_init()
541   */
542  static int
543  dcopy_list_init(dcopy_list_t *list, size_t node_size, offset_t link_offset)
544  {
545  	mutex_init(&list->dl_mutex, NULL, MUTEX_DRIVER, NULL);
546  	list_create(&list->dl_list, node_size, link_offset);
547  	list->dl_cnt = 0;
548  
549  	return (DCOPY_SUCCESS);
550  }
551  
552  
553  /*
554   * dcopy_list_fini()
555   */
556  static void
557  dcopy_list_fini(dcopy_list_t *list)
558  {
559  	list_destroy(&list->dl_list);
560  	mutex_destroy(&list->dl_mutex);
561  }
562  
563  
564  /*
565   * dcopy_list_push()
566   */
567  static void
568  dcopy_list_push(dcopy_list_t *list, void *list_node)
569  {
570  	mutex_enter(&list->dl_mutex);
571  	list_insert_tail(&list->dl_list, list_node);
572  	list->dl_cnt++;
573  	mutex_exit(&list->dl_mutex);
574  }
575  
576  
577  /*
578   * dcopy_list_pop()
579   */
580  static void *
581  dcopy_list_pop(dcopy_list_t *list)
582  {
583  	list_node_t *list_node;
584  
585  	mutex_enter(&list->dl_mutex);
586  	list_node = list_head(&list->dl_list);
587  	if (list_node == NULL) {
588  		mutex_exit(&list->dl_mutex);
589  		return (list_node);
590  	}
591  	list->dl_cnt--;
592  	list_remove(&list->dl_list, list_node);
593  	mutex_exit(&list->dl_mutex);
594  
595  	return (list_node);
596  }
597  
598  
599  /* *** DEVICE INTERFACE *** */
600  /*
601   * dcopy_device_register()
602   */
603  int
604  dcopy_device_register(void *device_private, dcopy_device_info_t *info,
605      dcopy_device_handle_t *handle)
606  {
607  	struct dcopy_channel_s *channel;
608  	struct dcopy_device_s *device;
609  	int e;
610  	int i;
611  
612  
613  	/* initialize the per device state */
614  	device = kmem_zalloc(sizeof (*device), KM_SLEEP);
615  	device->dc_device_private = device_private;
616  	device->dc_info = *info;
617  	device->dc_removing_cnt = 0;
618  	device->dc_cb = info->di_cb;
619  
620  	/*
621  	 * we have a per device channel list so we can remove a device in the
622  	 * future.
623  	 */
624  	e = dcopy_list_init(&device->dc_devchan_list,
625  	    sizeof (struct dcopy_channel_s),
626  	    offsetof(struct dcopy_channel_s, ch_devchan_list_node));
627  	if (e != DCOPY_SUCCESS) {
628  		goto registerfail_devchan;
629  	}
630  
631  	/*
632  	 * allocate state for each channel, allocate the channel,  and then add
633  	 * the devices dma channels to the devices channel list.
634  	 */
635  	for (i = 0; i < info->di_num_dma; i++) {
636  		channel = kmem_zalloc(sizeof (*channel), KM_SLEEP);
637  		channel->ch_device = device;
638  		channel->ch_removing = B_FALSE;
639  		channel->ch_ref_cnt = 0;
640  		channel->ch_cb = info->di_cb;
641  
642  		e = info->di_cb->cb_channel_alloc(device_private, channel,
643  		    DCOPY_SLEEP, dcopy_channel_size, &channel->ch_info,
644  		    &channel->ch_channel_private);
645  		if (e != DCOPY_SUCCESS) {
646  			kmem_free(channel, sizeof (*channel));
647  			goto registerfail_alloc;
648  		}
649  
650  		e = dcopy_stats_init(channel);
651  		if (e != DCOPY_SUCCESS) {
652  			info->di_cb->cb_channel_free(
653  			    &channel->ch_channel_private);
654  			kmem_free(channel, sizeof (*channel));
655  			goto registerfail_alloc;
656  		}
657  
658  		e = dcopy_list_init(&channel->ch_poll_list,
659  		    sizeof (struct dcopy_cmd_priv_s),
660  		    offsetof(struct dcopy_cmd_priv_s, pr_poll_list_node));
661  		if (e != DCOPY_SUCCESS) {
662  			dcopy_stats_fini(channel);
663  			info->di_cb->cb_channel_free(
664  			    &channel->ch_channel_private);
665  			kmem_free(channel, sizeof (*channel));
666  			goto registerfail_alloc;
667  		}
668  
669  		dcopy_list_push(&device->dc_devchan_list, channel);
670  	}
671  
672  	/* add the device to device list */
673  	dcopy_list_push(&dcopy_statep->d_device_list, device);
674  
675  	/*
676  	 * add the device's dma channels to the global channel list (where
677  	 * dcopy_alloc's come from)
678  	 */
679  	mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
680  	mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
681  	channel = list_head(&device->dc_devchan_list.dl_list);
682  	while (channel != NULL) {
683  		list_insert_tail(&dcopy_statep->d_globalchan_list.dl_list,
684  		    channel);
685  		dcopy_statep->d_globalchan_list.dl_cnt++;
686  		channel = list_next(&device->dc_devchan_list.dl_list, channel);
687  	}
688  	mutex_exit(&dcopy_statep->d_device_list.dl_mutex);
689  	mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);
690  
691  	*handle = device;
692  
693  	/* last call-back into kernel for dcopy KAPI enabled */
694  	uioa_dcopy_enable();
695  
696  	return (DCOPY_SUCCESS);
697  
698  registerfail_alloc:
699  	channel = list_head(&device->dc_devchan_list.dl_list);
700  	while (channel != NULL) {
701  		/* remove from the list */
702  		channel = dcopy_list_pop(&device->dc_devchan_list);
703  		ASSERT(channel != NULL);
704  
705  		dcopy_list_fini(&channel->ch_poll_list);
706  		dcopy_stats_fini(channel);
707  		info->di_cb->cb_channel_free(&channel->ch_channel_private);
708  		kmem_free(channel, sizeof (*channel));
709  	}
710  
711  	dcopy_list_fini(&device->dc_devchan_list);
712  registerfail_devchan:
713  	kmem_free(device, sizeof (*device));
714  
715  	return (DCOPY_FAILURE);
716  }
717  
718  
719  /*
720   * dcopy_device_unregister()
721   */
722  /*ARGSUSED*/
723  int
724  dcopy_device_unregister(dcopy_device_handle_t *handle)
725  {
726  	struct dcopy_channel_s *channel;
727  	dcopy_device_handle_t device;
728  	boolean_t device_busy;
729  
730  	/* first call-back into kernel for dcopy KAPI disable */
731  	uioa_dcopy_disable();
732  
733  	device = *handle;
734  	device_busy = B_FALSE;
735  
736  	/*
737  	 * remove the devices dma channels from the global channel list (where
738  	 * dcopy_alloc's come from)
739  	 */
740  	mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
741  	mutex_enter(&device->dc_devchan_list.dl_mutex);
742  	channel = list_head(&device->dc_devchan_list.dl_list);
743  	while (channel != NULL) {
744  		/*
745  		 * if the channel has outstanding allocs, mark it as having
746  		 * to be removed and increment the number of channels which
747  		 * need to be removed in the device state too.
748  		 */
749  		if (channel->ch_ref_cnt != 0) {
750  			channel->ch_removing = B_TRUE;
751  			device_busy = B_TRUE;
752  			device->dc_removing_cnt++;
753  		}
754  		dcopy_statep->d_globalchan_list.dl_cnt--;
755  		list_remove(&dcopy_statep->d_globalchan_list.dl_list, channel);
756  		channel = list_next(&device->dc_devchan_list.dl_list, channel);
757  	}
758  	mutex_exit(&device->dc_devchan_list.dl_mutex);
759  	mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);
760  
761  	/*
762  	 * if there are channels which still need to be removed, we will clean
763  	 * up the device state after they are freed up.
764  	 */
765  	if (device_busy) {
766  		return (DCOPY_PENDING);
767  	}
768  
769  	dcopy_device_cleanup(device, B_FALSE);
770  
771  	*handle = NULL;
772  	return (DCOPY_SUCCESS);
773  }
774  
775  
776  /*
777   * dcopy_device_cleanup()
778   */
779  static void
780  dcopy_device_cleanup(dcopy_device_handle_t device, boolean_t do_callback)
781  {
782  	struct dcopy_channel_s *channel;
783  
784  	/*
785  	 * remove all the channels in the device list, free them, and clean up
786  	 * the state.
787  	 */
788  	mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
789  	channel = list_head(&device->dc_devchan_list.dl_list);
790  	while (channel != NULL) {
791  		device->dc_devchan_list.dl_cnt--;
792  		list_remove(&device->dc_devchan_list.dl_list, channel);
793  		dcopy_list_fini(&channel->ch_poll_list);
794  		dcopy_stats_fini(channel);
795  		channel->ch_cb->cb_channel_free(&channel->ch_channel_private);
796  		kmem_free(channel, sizeof (*channel));
797  		channel = list_head(&device->dc_devchan_list.dl_list);
798  	}
799  
800  	/* remove it from the list of devices */
801  	list_remove(&dcopy_statep->d_device_list.dl_list, device);
802  
803  	mutex_exit(&dcopy_statep->d_device_list.dl_mutex);
804  
805  	/*
806  	 * notify the DMA device driver that the device is free to be
807  	 * detached.
808  	 */
809  	if (do_callback) {
810  		device->dc_cb->cb_unregister_complete(
811  		    device->dc_device_private, DCOPY_SUCCESS);
812  	}
813  
814  	dcopy_list_fini(&device->dc_devchan_list);
815  	kmem_free(device, sizeof (*device));
816  }
817  
818  
819  /*
820   * dcopy_device_channel_notify()
821   */
822  /*ARGSUSED*/
823  void
824  dcopy_device_channel_notify(dcopy_handle_t handle, int status)
825  {
826  	struct dcopy_channel_s *channel;
827  	dcopy_list_t *poll_list;
828  	dcopy_cmd_priv_t priv;
829  	int e;
830  
831  
832  	ASSERT(status == DCOPY_COMPLETION);
833  	channel = handle;
834  
835  	poll_list = &channel->ch_poll_list;
836  
837  	/*
838  	 * when we get a completion notification from the device, go through
839  	 * all of the commands blocking on this channel and see if they have
840  	 * completed. Remove the command and wake up the block thread if they
841  	 * have. Once we hit a command which is still pending, we are done
842  	 * polling since commands in a channel complete in order.
843  	 */
844  	mutex_enter(&poll_list->dl_mutex);
845  	if (poll_list->dl_cnt != 0) {
846  		priv = list_head(&poll_list->dl_list);
847  		while (priv != NULL) {
848  			atomic_inc_64(&channel->
849  			    ch_stat.cs_notify_poll.value.ui64);
850  			e = channel->ch_cb->cb_cmd_poll(
851  			    channel->ch_channel_private,
852  			    priv->pr_cmd);
853  			if (e == DCOPY_PENDING) {
854  				atomic_inc_64(&channel->
855  				    ch_stat.cs_notify_pending.value.ui64);
856  				break;
857  			}
858  
859  			poll_list->dl_cnt--;
860  			list_remove(&poll_list->dl_list, priv);
861  
862  			mutex_enter(&priv->pr_mutex);
863  			priv->pr_wait = B_FALSE;
864  			cv_signal(&priv->pr_cv);
865  			mutex_exit(&priv->pr_mutex);
866  
867  			priv = list_head(&poll_list->dl_list);
868  		}
869  	}
870  
871  	mutex_exit(&poll_list->dl_mutex);
872  }
873  
874  
875  /*
876   * dcopy_stats_init()
877   */
878  static int
879  dcopy_stats_init(dcopy_handle_t channel)
880  {
881  #define	CHANSTRSIZE	20
882  	char chanstr[CHANSTRSIZE];
883  	dcopy_stats_t *stats;
884  	int instance;
885  	char *name;
886  
887  
888  	stats = &channel->ch_stat;
889  	name = (char *)ddi_driver_name(channel->ch_device->dc_info.di_dip);
890  	instance = ddi_get_instance(channel->ch_device->dc_info.di_dip);
891  
892  	(void) snprintf(chanstr, CHANSTRSIZE, "channel%d",
893  	    (uint32_t)channel->ch_info.qc_chan_num);
894  
895  	channel->ch_kstat = kstat_create(name, instance, chanstr, "misc",
896  	    KSTAT_TYPE_NAMED, sizeof (dcopy_stats_t) / sizeof (kstat_named_t),
897  	    KSTAT_FLAG_VIRTUAL);
898  	if (channel->ch_kstat == NULL) {
899  		return (DCOPY_FAILURE);
900  	}
901  	channel->ch_kstat->ks_data = stats;
902  
903  	kstat_named_init(&stats->cs_bytes_xfer, "bytes_xfer",
904  	    KSTAT_DATA_UINT64);
905  	kstat_named_init(&stats->cs_cmd_alloc, "cmd_alloc",
906  	    KSTAT_DATA_UINT64);
907  	kstat_named_init(&stats->cs_cmd_post, "cmd_post",
908  	    KSTAT_DATA_UINT64);
909  	kstat_named_init(&stats->cs_cmd_poll, "cmd_poll",
910  	    KSTAT_DATA_UINT64);
911  	kstat_named_init(&stats->cs_notify_poll, "notify_poll",
912  	    KSTAT_DATA_UINT64);
913  	kstat_named_init(&stats->cs_notify_pending, "notify_pending",
914  	    KSTAT_DATA_UINT64);
915  	kstat_named_init(&stats->cs_id, "id",
916  	    KSTAT_DATA_UINT64);
917  	kstat_named_init(&stats->cs_capabilities, "capabilities",
918  	    KSTAT_DATA_UINT64);
919  
920  	kstat_install(channel->ch_kstat);
921  
922  	channel->ch_stat.cs_id.value.ui64 = channel->ch_info.qc_id;
923  	channel->ch_stat.cs_capabilities.value.ui64 =
924  	    channel->ch_info.qc_capabilities;
925  
926  	return (DCOPY_SUCCESS);
927  }
928  
929  
930  /*
931   * dcopy_stats_fini()
932   */
933  static void
934  dcopy_stats_fini(dcopy_handle_t channel)
935  {
936  	kstat_delete(channel->ch_kstat);
937  }
938  /* *** END OF DEVICE INTERFACE *** */
939