xref: /freebsd/sys/dev/hyperv/vmbus/vmbus_chan.c (revision cddbc3b40812213ff00041f79174cac0be360a2a)
1 /*-
2  * Copyright (c) 2009-2012,2016 Microsoft Corp.
3  * Copyright (c) 2012 NetApp Inc.
4  * Copyright (c) 2012 Citrix Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/callout.h>
35 #include <sys/kernel.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/mutex.h>
39 #include <sys/smp.h>
40 #include <sys/sysctl.h>
41 #include <sys/systm.h>
42 
43 #include <machine/atomic.h>
44 #include <machine/stdarg.h>
45 
46 #include <dev/hyperv/include/hyperv_busdma.h>
47 #include <dev/hyperv/include/vmbus_xact.h>
48 #include <dev/hyperv/vmbus/hyperv_var.h>
49 #include <dev/hyperv/vmbus/vmbus_reg.h>
50 #include <dev/hyperv/vmbus/vmbus_var.h>
51 #include <dev/hyperv/vmbus/vmbus_brvar.h>
52 #include <dev/hyperv/vmbus/vmbus_chanvar.h>
53 
54 struct vmbus_chan_pollarg {
55 	struct vmbus_channel	*poll_chan;
56 	u_int			poll_hz;
57 };
58 
59 static void			vmbus_chan_update_evtflagcnt(
60 				    struct vmbus_softc *,
61 				    const struct vmbus_channel *);
62 static int			vmbus_chan_close_internal(
63 				    struct vmbus_channel *);
64 static int			vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS);
65 static void			vmbus_chan_sysctl_create(
66 				    struct vmbus_channel *);
67 static struct vmbus_channel	*vmbus_chan_alloc(struct vmbus_softc *);
68 static void			vmbus_chan_free(struct vmbus_channel *);
69 static int			vmbus_chan_add(struct vmbus_channel *);
70 static void			vmbus_chan_cpu_default(struct vmbus_channel *);
71 static int			vmbus_chan_release(struct vmbus_channel *);
72 static void			vmbus_chan_set_chmap(struct vmbus_channel *);
73 static void			vmbus_chan_clear_chmap(struct vmbus_channel *);
74 static void			vmbus_chan_detach(struct vmbus_channel *);
75 static bool			vmbus_chan_wait_revoke(
76 				    const struct vmbus_channel *, bool);
77 static void			vmbus_chan_poll_timeout(void *);
78 static bool			vmbus_chan_poll_cancel_intq(
79 				    struct vmbus_channel *);
80 static void			vmbus_chan_poll_cancel(struct vmbus_channel *);
81 
82 static void			vmbus_chan_ins_prilist(struct vmbus_softc *,
83 				    struct vmbus_channel *);
84 static void			vmbus_chan_rem_prilist(struct vmbus_softc *,
85 				    struct vmbus_channel *);
86 static void			vmbus_chan_ins_list(struct vmbus_softc *,
87 				    struct vmbus_channel *);
88 static void			vmbus_chan_rem_list(struct vmbus_softc *,
89 				    struct vmbus_channel *);
90 static void			vmbus_chan_ins_sublist(struct vmbus_channel *,
91 				    struct vmbus_channel *);
92 static void			vmbus_chan_rem_sublist(struct vmbus_channel *,
93 				    struct vmbus_channel *);
94 
95 static void			vmbus_chan_task(void *, int);
96 static void			vmbus_chan_task_nobatch(void *, int);
97 static void			vmbus_chan_poll_task(void *, int);
98 static void			vmbus_chan_clrchmap_task(void *, int);
99 static void			vmbus_chan_pollcfg_task(void *, int);
100 static void			vmbus_chan_polldis_task(void *, int);
101 static void			vmbus_chan_poll_cancel_task(void *, int);
102 static void			vmbus_prichan_attach_task(void *, int);
103 static void			vmbus_subchan_attach_task(void *, int);
104 static void			vmbus_prichan_detach_task(void *, int);
105 static void			vmbus_subchan_detach_task(void *, int);
106 
107 static void			vmbus_chan_msgproc_choffer(struct vmbus_softc *,
108 				    const struct vmbus_message *);
109 static void			vmbus_chan_msgproc_chrescind(
110 				    struct vmbus_softc *,
111 				    const struct vmbus_message *);
112 
113 static int			vmbus_chan_printf(const struct vmbus_channel *,
114 				    const char *, ...) __printflike(2, 3);
115 
116 /*
117  * Vmbus channel message processing.
118  */
119 static const vmbus_chanmsg_proc_t
120 vmbus_chan_msgprocs[VMBUS_CHANMSG_TYPE_MAX] = {
121 	VMBUS_CHANMSG_PROC(CHOFFER,	vmbus_chan_msgproc_choffer),
122 	VMBUS_CHANMSG_PROC(CHRESCIND,	vmbus_chan_msgproc_chrescind),
123 
124 	VMBUS_CHANMSG_PROC_WAKEUP(CHOPEN_RESP),
125 	VMBUS_CHANMSG_PROC_WAKEUP(GPADL_CONNRESP),
126 	VMBUS_CHANMSG_PROC_WAKEUP(GPADL_DISCONNRESP)
127 };
128 
129 /*
130  * Notify host that there are data pending on our TX bufring.
131  */
132 static __inline void
133 vmbus_chan_signal_tx(const struct vmbus_channel *chan)
134 {
135 	atomic_set_long(chan->ch_evtflag, chan->ch_evtflag_mask);
136 	if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF)
137 		atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask);
138 	else
139 		hypercall_signal_event(chan->ch_monprm_dma.hv_paddr);
140 }
141 
142 static void
143 vmbus_chan_ins_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan)
144 {
145 
146 	mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED);
147 	if (atomic_testandset_int(&chan->ch_stflags,
148 	    VMBUS_CHAN_ST_ONPRIL_SHIFT))
149 		panic("channel is already on the prilist");
150 	TAILQ_INSERT_TAIL(&sc->vmbus_prichans, chan, ch_prilink);
151 }
152 
153 static void
154 vmbus_chan_rem_prilist(struct vmbus_softc *sc, struct vmbus_channel *chan)
155 {
156 
157 	mtx_assert(&sc->vmbus_prichan_lock, MA_OWNED);
158 	if (atomic_testandclear_int(&chan->ch_stflags,
159 	    VMBUS_CHAN_ST_ONPRIL_SHIFT) == 0)
160 		panic("channel is not on the prilist");
161 	TAILQ_REMOVE(&sc->vmbus_prichans, chan, ch_prilink);
162 }
163 
164 static void
165 vmbus_chan_ins_sublist(struct vmbus_channel *prichan,
166     struct vmbus_channel *chan)
167 {
168 
169 	mtx_assert(&prichan->ch_subchan_lock, MA_OWNED);
170 
171 	if (atomic_testandset_int(&chan->ch_stflags,
172 	    VMBUS_CHAN_ST_ONSUBL_SHIFT))
173 		panic("channel is already on the sublist");
174 	TAILQ_INSERT_TAIL(&prichan->ch_subchans, chan, ch_sublink);
175 
176 	/* Bump sub-channel count. */
177 	prichan->ch_subchan_cnt++;
178 }
179 
180 static void
181 vmbus_chan_rem_sublist(struct vmbus_channel *prichan,
182     struct vmbus_channel *chan)
183 {
184 
185 	mtx_assert(&prichan->ch_subchan_lock, MA_OWNED);
186 
187 	KASSERT(prichan->ch_subchan_cnt > 0,
188 	    ("invalid subchan_cnt %d", prichan->ch_subchan_cnt));
189 	prichan->ch_subchan_cnt--;
190 
191 	if (atomic_testandclear_int(&chan->ch_stflags,
192 	    VMBUS_CHAN_ST_ONSUBL_SHIFT) == 0)
193 		panic("channel is not on the sublist");
194 	TAILQ_REMOVE(&prichan->ch_subchans, chan, ch_sublink);
195 }
196 
197 static void
198 vmbus_chan_ins_list(struct vmbus_softc *sc, struct vmbus_channel *chan)
199 {
200 
201 	mtx_assert(&sc->vmbus_chan_lock, MA_OWNED);
202 	if (atomic_testandset_int(&chan->ch_stflags,
203 	    VMBUS_CHAN_ST_ONLIST_SHIFT))
204 		panic("channel is already on the list");
205 	TAILQ_INSERT_TAIL(&sc->vmbus_chans, chan, ch_link);
206 }
207 
208 static void
209 vmbus_chan_rem_list(struct vmbus_softc *sc, struct vmbus_channel *chan)
210 {
211 
212 	mtx_assert(&sc->vmbus_chan_lock, MA_OWNED);
213 	if (atomic_testandclear_int(&chan->ch_stflags,
214 	    VMBUS_CHAN_ST_ONLIST_SHIFT) == 0)
215 		panic("channel is not on the list");
216 	TAILQ_REMOVE(&sc->vmbus_chans, chan, ch_link);
217 }
218 
219 static int
220 vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS)
221 {
222 	struct vmbus_channel *chan = arg1;
223 	int mnf = 0;
224 
225 	if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF)
226 		mnf = 1;
227 	return sysctl_handle_int(oidp, &mnf, 0, req);
228 }
229 
230 static void
231 vmbus_chan_sysctl_create(struct vmbus_channel *chan)
232 {
233 	struct sysctl_oid *ch_tree, *chid_tree, *br_tree;
234 	struct sysctl_ctx_list *ctx;
235 	uint32_t ch_id;
236 	char name[16];
237 
238 	/*
239 	 * Add sysctl nodes related to this channel to this
240 	 * channel's sysctl ctx, so that they can be destroyed
241 	 * independently upon close of this channel, which can
242 	 * happen even if the device is not detached.
243 	 */
244 	ctx = &chan->ch_sysctl_ctx;
245 	sysctl_ctx_init(ctx);
246 
247 	/*
248 	 * Create dev.NAME.UNIT.channel tree.
249 	 */
250 	ch_tree = SYSCTL_ADD_NODE(ctx,
251 	    SYSCTL_CHILDREN(device_get_sysctl_tree(chan->ch_dev)),
252 	    OID_AUTO, "channel", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
253 	if (ch_tree == NULL)
254 		return;
255 
256 	/*
257 	 * Create dev.NAME.UNIT.channel.CHANID tree.
258 	 */
259 	if (VMBUS_CHAN_ISPRIMARY(chan))
260 		ch_id = chan->ch_id;
261 	else
262 		ch_id = chan->ch_prichan->ch_id;
263 	snprintf(name, sizeof(name), "%d", ch_id);
264 	chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree),
265 	    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
266 	if (chid_tree == NULL)
267 		return;
268 
269 	if (!VMBUS_CHAN_ISPRIMARY(chan)) {
270 		/*
271 		 * Create dev.NAME.UNIT.channel.CHANID.sub tree.
272 		 */
273 		ch_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree),
274 		    OID_AUTO, "sub", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
275 		if (ch_tree == NULL)
276 			return;
277 
278 		/*
279 		 * Create dev.NAME.UNIT.channel.CHANID.sub.SUBIDX tree.
280 		 *
281 		 * NOTE:
282 		 * chid_tree is changed to this new sysctl tree.
283 		 */
284 		snprintf(name, sizeof(name), "%d", chan->ch_subidx);
285 		chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree),
286 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
287 		if (chid_tree == NULL)
288 			return;
289 
290 		SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO,
291 		    "chanid", CTLFLAG_RD, &chan->ch_id, 0, "channel id");
292 	}
293 
294 	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO,
295 	    "cpu", CTLFLAG_RD, &chan->ch_cpuid, 0, "owner CPU id");
296 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO,
297 	    "mnf", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE,
298 	    chan, 0, vmbus_chan_sysctl_mnf, "I",
299 	    "has monitor notification facilities");
300 
301 	br_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO,
302 	    "br", CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
303 	if (br_tree != NULL) {
304 		/*
305 		 * Create sysctl tree for RX bufring.
306 		 */
307 		vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx");
308 		/*
309 		 * Create sysctl tree for TX bufring.
310 		 */
311 		vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx");
312 	}
313 }
314 
315 int
316 vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size,
317     const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg)
318 {
319 	struct vmbus_chan_br cbr;
320 	int error;
321 
322 	/*
323 	 * Allocate the TX+RX bufrings.
324 	 */
325 	KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated"));
326 	chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev),
327 	    PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma,
328 	    BUS_DMA_WAITOK);
329 	if (chan->ch_bufring == NULL) {
330 		vmbus_chan_printf(chan, "bufring allocation failed\n");
331 		return (ENOMEM);
332 	}
333 
334 	cbr.cbr = chan->ch_bufring;
335 	cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr;
336 	cbr.cbr_txsz = txbr_size;
337 	cbr.cbr_rxsz = rxbr_size;
338 
339 	error = vmbus_chan_open_br(chan, &cbr, udata, udlen, cb, cbarg);
340 	if (error) {
341 		if (error == EISCONN) {
342 			/*
343 			 * XXX
344 			 * The bufring GPADL is still connected; abandon
345 			 * this bufring, instead of having mysterious
346 			 * crash or trashed data later on.
347 			 */
348 			vmbus_chan_printf(chan, "chan%u bufring GPADL "
349 			    "is still connected upon channel open error; "
350 			    "leak %d bytes memory\n", chan->ch_id,
351 			    txbr_size + rxbr_size);
352 		} else {
353 			hyperv_dmamem_free(&chan->ch_bufring_dma,
354 			    chan->ch_bufring);
355 		}
356 		chan->ch_bufring = NULL;
357 	}
358 	return (error);
359 }
360 
361 int
362 vmbus_chan_open_br(struct vmbus_channel *chan, const struct vmbus_chan_br *cbr,
363     const void *udata, int udlen, vmbus_chan_callback_t cb, void *cbarg)
364 {
365 	struct vmbus_softc *sc = chan->ch_vmbus;
366 	const struct vmbus_message *msg;
367 	struct vmbus_chanmsg_chopen *req;
368 	struct vmbus_msghc *mh;
369 	uint32_t status;
370 	int error, txbr_size, rxbr_size;
371 	task_fn_t *task_fn;
372 	uint8_t *br;
373 
374 	if (udlen > VMBUS_CHANMSG_CHOPEN_UDATA_SIZE) {
375 		vmbus_chan_printf(chan,
376 		    "invalid udata len %d for chan%u\n", udlen, chan->ch_id);
377 		return (EINVAL);
378 	}
379 
380 	br = cbr->cbr;
381 	txbr_size = cbr->cbr_txsz;
382 	rxbr_size = cbr->cbr_rxsz;
383 	KASSERT((txbr_size & PAGE_MASK) == 0,
384 	    ("send bufring size is not multiple page"));
385 	KASSERT((rxbr_size & PAGE_MASK) == 0,
386 	    ("recv bufring size is not multiple page"));
387 	KASSERT((cbr->cbr_paddr & PAGE_MASK) == 0,
388 	    ("bufring is not page aligned"));
389 
390 	/*
391 	 * Zero out the TX/RX bufrings, in case that they were used before.
392 	 */
393 	memset(br, 0, txbr_size + rxbr_size);
394 
395 	if (atomic_testandset_int(&chan->ch_stflags,
396 	    VMBUS_CHAN_ST_OPENED_SHIFT))
397 		panic("double-open chan%u", chan->ch_id);
398 
399 	chan->ch_cb = cb;
400 	chan->ch_cbarg = cbarg;
401 
402 	vmbus_chan_update_evtflagcnt(sc, chan);
403 
404 	chan->ch_tq = VMBUS_PCPU_GET(chan->ch_vmbus, event_tq, chan->ch_cpuid);
405 	if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD)
406 		task_fn = vmbus_chan_task;
407 	else
408 		task_fn = vmbus_chan_task_nobatch;
409 	TASK_INIT(&chan->ch_task, 0, task_fn, chan);
410 
411 	/* TX bufring comes first */
412 	vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size);
413 	/* RX bufring immediately follows TX bufring */
414 	vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size);
415 
416 	/* Create sysctl tree for this channel */
417 	vmbus_chan_sysctl_create(chan);
418 
419 	/*
420 	 * Connect the bufrings, both RX and TX, to this channel.
421 	 */
422 	error = vmbus_chan_gpadl_connect(chan, cbr->cbr_paddr,
423 	    txbr_size + rxbr_size, &chan->ch_bufring_gpadl);
424 	if (error) {
425 		vmbus_chan_printf(chan,
426 		    "failed to connect bufring GPADL to chan%u\n", chan->ch_id);
427 		goto failed;
428 	}
429 
430 	/*
431 	 * Install this channel, before it is opened, but after everything
432 	 * else has been setup.
433 	 */
434 	vmbus_chan_set_chmap(chan);
435 
436 	/*
437 	 * Open channel w/ the bufring GPADL on the target CPU.
438 	 */
439 	mh = vmbus_msghc_get(sc, sizeof(*req));
440 	if (mh == NULL) {
441 		vmbus_chan_printf(chan,
442 		    "can not get msg hypercall for chopen(chan%u)\n",
443 		    chan->ch_id);
444 		error = ENXIO;
445 		goto failed;
446 	}
447 
448 	req = vmbus_msghc_dataptr(mh);
449 	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHOPEN;
450 	req->chm_chanid = chan->ch_id;
451 	req->chm_openid = chan->ch_id;
452 	req->chm_gpadl = chan->ch_bufring_gpadl;
453 	req->chm_vcpuid = chan->ch_vcpuid;
454 	req->chm_txbr_pgcnt = txbr_size >> PAGE_SHIFT;
455 	if (udlen > 0)
456 		memcpy(req->chm_udata, udata, udlen);
457 
458 	error = vmbus_msghc_exec(sc, mh);
459 	if (error) {
460 		vmbus_chan_printf(chan,
461 		    "chopen(chan%u) msg hypercall exec failed: %d\n",
462 		    chan->ch_id, error);
463 		vmbus_msghc_put(sc, mh);
464 		goto failed;
465 	}
466 
467 	for (;;) {
468 		msg = vmbus_msghc_poll_result(sc, mh);
469 		if (msg != NULL)
470 			break;
471 		if (vmbus_chan_is_revoked(chan)) {
472 			int i;
473 
474 			/*
475 			 * NOTE:
476 			 * Hypervisor does _not_ send response CHOPEN to
477 			 * a revoked channel.
478 			 */
479 			vmbus_chan_printf(chan,
480 			    "chan%u is revoked, when it is being opened\n",
481 			    chan->ch_id);
482 
483 			/*
484 			 * XXX
485 			 * Add extra delay before cancel the hypercall
486 			 * execution; mainly to close any possible
487 			 * CHRESCIND and CHOPEN_RESP races on the
488 			 * hypervisor side.
489 			 */
490 #define REVOKE_LINGER	100
491 			for (i = 0; i < REVOKE_LINGER; ++i) {
492 				msg = vmbus_msghc_poll_result(sc, mh);
493 				if (msg != NULL)
494 					break;
495 				pause("rchopen", 1);
496 			}
497 #undef REVOKE_LINGER
498 			if (msg == NULL)
499 				vmbus_msghc_exec_cancel(sc, mh);
500 			break;
501 		}
502 		pause("chopen", 1);
503 	}
504 	if (msg != NULL) {
505 		status = ((const struct vmbus_chanmsg_chopen_resp *)
506 		    msg->msg_data)->chm_status;
507 	} else {
508 		/* XXX any non-0 value is ok here. */
509 		status = 0xff;
510 	}
511 
512 	vmbus_msghc_put(sc, mh);
513 
514 	if (status == 0) {
515 		if (bootverbose)
516 			vmbus_chan_printf(chan, "chan%u opened\n", chan->ch_id);
517 		return (0);
518 	}
519 
520 	vmbus_chan_printf(chan, "failed to open chan%u\n", chan->ch_id);
521 	error = ENXIO;
522 
523 failed:
524 	sysctl_ctx_free(&chan->ch_sysctl_ctx);
525 	vmbus_chan_clear_chmap(chan);
526 	if (chan->ch_bufring_gpadl != 0) {
527 		int error1;
528 
529 		error1 = vmbus_chan_gpadl_disconnect(chan,
530 		    chan->ch_bufring_gpadl);
531 		if (error1) {
532 			/*
533 			 * Give caller a hint that the bufring GPADL is still
534 			 * connected.
535 			 */
536 			error = EISCONN;
537 		}
538 		chan->ch_bufring_gpadl = 0;
539 	}
540 	atomic_clear_int(&chan->ch_stflags, VMBUS_CHAN_ST_OPENED);
541 	return (error);
542 }
543 
544 int
545 vmbus_chan_gpadl_connect(struct vmbus_channel *chan, bus_addr_t paddr,
546     int size, uint32_t *gpadl0)
547 {
548 	struct vmbus_softc *sc = chan->ch_vmbus;
549 	struct vmbus_msghc *mh;
550 	struct vmbus_chanmsg_gpadl_conn *req;
551 	const struct vmbus_message *msg;
552 	size_t reqsz;
553 	uint32_t gpadl, status;
554 	int page_count, range_len, i, cnt, error;
555 	uint64_t page_id;
556 
557 	KASSERT(*gpadl0 == 0, ("GPADL is not zero"));
558 
559 	/*
560 	 * Preliminary checks.
561 	 */
562 
563 	KASSERT((size & PAGE_MASK) == 0,
564 	    ("invalid GPA size %d, not multiple page size", size));
565 	page_count = size >> PAGE_SHIFT;
566 
567 	KASSERT((paddr & PAGE_MASK) == 0,
568 	    ("GPA is not page aligned %jx", (uintmax_t)paddr));
569 	page_id = paddr >> PAGE_SHIFT;
570 
571 	range_len = __offsetof(struct vmbus_gpa_range, gpa_page[page_count]);
572 	/*
573 	 * We don't support multiple GPA ranges.
574 	 */
575 	if (range_len > UINT16_MAX) {
576 		vmbus_chan_printf(chan, "GPA too large, %d pages\n",
577 		    page_count);
578 		return EOPNOTSUPP;
579 	}
580 
581 	/*
582 	 * Allocate GPADL id.
583 	 */
584 	gpadl = vmbus_gpadl_alloc(sc);
585 
586 	/*
587 	 * Connect this GPADL to the target channel.
588 	 *
589 	 * NOTE:
590 	 * Since each message can only hold small set of page
591 	 * addresses, several messages may be required to
592 	 * complete the connection.
593 	 */
594 	if (page_count > VMBUS_CHANMSG_GPADL_CONN_PGMAX)
595 		cnt = VMBUS_CHANMSG_GPADL_CONN_PGMAX;
596 	else
597 		cnt = page_count;
598 	page_count -= cnt;
599 
600 	reqsz = __offsetof(struct vmbus_chanmsg_gpadl_conn,
601 	    chm_range.gpa_page[cnt]);
602 	mh = vmbus_msghc_get(sc, reqsz);
603 	if (mh == NULL) {
604 		vmbus_chan_printf(chan,
605 		    "can not get msg hypercall for gpadl_conn(chan%u)\n",
606 		    chan->ch_id);
607 		return EIO;
608 	}
609 
610 	req = vmbus_msghc_dataptr(mh);
611 	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_CONN;
612 	req->chm_chanid = chan->ch_id;
613 	req->chm_gpadl = gpadl;
614 	req->chm_range_len = range_len;
615 	req->chm_range_cnt = 1;
616 	req->chm_range.gpa_len = size;
617 	req->chm_range.gpa_ofs = 0;
618 	for (i = 0; i < cnt; ++i)
619 		req->chm_range.gpa_page[i] = page_id++;
620 
621 	error = vmbus_msghc_exec(sc, mh);
622 	if (error) {
623 		vmbus_chan_printf(chan,
624 		    "gpadl_conn(chan%u) msg hypercall exec failed: %d\n",
625 		    chan->ch_id, error);
626 		vmbus_msghc_put(sc, mh);
627 		return error;
628 	}
629 
630 	while (page_count > 0) {
631 		struct vmbus_chanmsg_gpadl_subconn *subreq;
632 
633 		if (page_count > VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX)
634 			cnt = VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX;
635 		else
636 			cnt = page_count;
637 		page_count -= cnt;
638 
639 		reqsz = __offsetof(struct vmbus_chanmsg_gpadl_subconn,
640 		    chm_gpa_page[cnt]);
641 		vmbus_msghc_reset(mh, reqsz);
642 
643 		subreq = vmbus_msghc_dataptr(mh);
644 		subreq->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_SUBCONN;
645 		subreq->chm_gpadl = gpadl;
646 		for (i = 0; i < cnt; ++i)
647 			subreq->chm_gpa_page[i] = page_id++;
648 
649 		vmbus_msghc_exec_noresult(mh);
650 	}
651 	KASSERT(page_count == 0, ("invalid page count %d", page_count));
652 
653 	msg = vmbus_msghc_wait_result(sc, mh);
654 	status = ((const struct vmbus_chanmsg_gpadl_connresp *)
655 	    msg->msg_data)->chm_status;
656 
657 	vmbus_msghc_put(sc, mh);
658 
659 	if (status != 0) {
660 		vmbus_chan_printf(chan, "gpadl_conn(chan%u) failed: %u\n",
661 		    chan->ch_id, status);
662 		return EIO;
663 	}
664 
665 	/* Done; commit the GPADL id. */
666 	*gpadl0 = gpadl;
667 	if (bootverbose) {
668 		vmbus_chan_printf(chan, "gpadl_conn(chan%u) succeeded\n",
669 		    chan->ch_id);
670 	}
671 	return 0;
672 }
673 
674 static bool
675 vmbus_chan_wait_revoke(const struct vmbus_channel *chan, bool can_sleep)
676 {
677 #define WAIT_COUNT	200	/* 200ms */
678 
679 	int i;
680 
681 	for (i = 0; i < WAIT_COUNT; ++i) {
682 		if (vmbus_chan_is_revoked(chan))
683 			return (true);
684 		if (can_sleep)
685 			pause("wchrev", 1);
686 		else
687 			DELAY(1000);
688 	}
689 	return (false);
690 
691 #undef WAIT_COUNT
692 }
693 
694 /*
695  * Disconnect the GPA from the target channel
696  */
697 int
698 vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, uint32_t gpadl)
699 {
700 	struct vmbus_softc *sc = chan->ch_vmbus;
701 	struct vmbus_msghc *mh;
702 	struct vmbus_chanmsg_gpadl_disconn *req;
703 	int error;
704 
705 	KASSERT(gpadl != 0, ("GPADL is zero"));
706 
707 	mh = vmbus_msghc_get(sc, sizeof(*req));
708 	if (mh == NULL) {
709 		vmbus_chan_printf(chan,
710 		    "can not get msg hypercall for gpadl_disconn(chan%u)\n",
711 		    chan->ch_id);
712 		return (EBUSY);
713 	}
714 
715 	req = vmbus_msghc_dataptr(mh);
716 	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_GPADL_DISCONN;
717 	req->chm_chanid = chan->ch_id;
718 	req->chm_gpadl = gpadl;
719 
720 	error = vmbus_msghc_exec(sc, mh);
721 	if (error) {
722 		vmbus_msghc_put(sc, mh);
723 
724 		if (vmbus_chan_wait_revoke(chan, true)) {
725 			/*
726 			 * Error is benign; this channel is revoked,
727 			 * so this GPADL will not be touched anymore.
728 			 */
729 			vmbus_chan_printf(chan,
730 			    "gpadl_disconn(revoked chan%u) msg hypercall "
731 			    "exec failed: %d\n", chan->ch_id, error);
732 			return (0);
733 		}
734 		vmbus_chan_printf(chan,
735 		    "gpadl_disconn(chan%u) msg hypercall exec failed: %d\n",
736 		    chan->ch_id, error);
737 		return (error);
738 	}
739 
740 	vmbus_msghc_wait_result(sc, mh);
741 	/* Discard result; no useful information */
742 	vmbus_msghc_put(sc, mh);
743 
744 	return (0);
745 }
746 
747 static void
748 vmbus_chan_detach(struct vmbus_channel *chan)
749 {
750 	int refs;
751 
752 	KASSERT(chan->ch_refs > 0, ("chan%u: invalid refcnt %d",
753 	    chan->ch_id, chan->ch_refs));
754 	refs = atomic_fetchadd_int(&chan->ch_refs, -1);
755 #ifdef INVARIANTS
756 	if (VMBUS_CHAN_ISPRIMARY(chan)) {
757 		KASSERT(refs == 1, ("chan%u: invalid refcnt %d for prichan",
758 		    chan->ch_id, refs + 1));
759 	}
760 #endif
761 	if (refs == 1) {
762 		/*
763 		 * Detach the target channel.
764 		 */
765 		if (bootverbose) {
766 			vmbus_chan_printf(chan, "chan%u detached\n",
767 			    chan->ch_id);
768 		}
769 		taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task);
770 	}
771 }
772 
773 static void
774 vmbus_chan_clrchmap_task(void *xchan, int pending __unused)
775 {
776 	struct vmbus_channel *chan = xchan;
777 
778 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL;
779 }
780 
781 static void
782 vmbus_chan_clear_chmap(struct vmbus_channel *chan)
783 {
784 	struct task chmap_task;
785 
786 	TASK_INIT(&chmap_task, 0, vmbus_chan_clrchmap_task, chan);
787 	vmbus_chan_run_task(chan, &chmap_task);
788 }
789 
790 static void
791 vmbus_chan_set_chmap(struct vmbus_channel *chan)
792 {
793 	__compiler_membar();
794 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan;
795 }
796 
797 static void
798 vmbus_chan_poll_cancel_task(void *xchan, int pending __unused)
799 {
800 
801 	vmbus_chan_poll_cancel_intq(xchan);
802 }
803 
804 static void
805 vmbus_chan_poll_cancel(struct vmbus_channel *chan)
806 {
807 	struct task poll_cancel;
808 
809 	TASK_INIT(&poll_cancel, 0, vmbus_chan_poll_cancel_task, chan);
810 	vmbus_chan_run_task(chan, &poll_cancel);
811 }
812 
813 static int
814 vmbus_chan_close_internal(struct vmbus_channel *chan)
815 {
816 	struct vmbus_softc *sc = chan->ch_vmbus;
817 	struct vmbus_msghc *mh;
818 	struct vmbus_chanmsg_chclose *req;
819 	uint32_t old_stflags;
820 	int error;
821 
822 	/*
823 	 * NOTE:
824 	 * Sub-channels are closed upon their primary channel closing,
825 	 * so they can be closed even before they are opened.
826 	 */
827 	for (;;) {
828 		old_stflags = chan->ch_stflags;
829 		if (atomic_cmpset_int(&chan->ch_stflags, old_stflags,
830 		    old_stflags & ~VMBUS_CHAN_ST_OPENED))
831 			break;
832 	}
833 	if ((old_stflags & VMBUS_CHAN_ST_OPENED) == 0) {
834 		/* Not opened yet; done */
835 		if (bootverbose) {
836 			vmbus_chan_printf(chan, "chan%u not opened\n",
837 			    chan->ch_id);
838 		}
839 		return (0);
840 	}
841 
842 	/*
843 	 * Free this channel's sysctl tree attached to its device's
844 	 * sysctl tree.
845 	 */
846 	sysctl_ctx_free(&chan->ch_sysctl_ctx);
847 
848 	/*
849 	 * Cancel polling, if it is enabled.
850 	 */
851 	vmbus_chan_poll_cancel(chan);
852 
853 	/*
854 	 * NOTE:
855 	 * Order is critical.  This channel _must_ be uninstalled first,
856 	 * else the channel task may be enqueued by the IDT after it has
857 	 * been drained.
858 	 */
859 	vmbus_chan_clear_chmap(chan);
860 	taskqueue_drain(chan->ch_tq, &chan->ch_task);
861 	chan->ch_tq = NULL;
862 
863 	/*
864 	 * Close this channel.
865 	 */
866 	mh = vmbus_msghc_get(sc, sizeof(*req));
867 	if (mh == NULL) {
868 		vmbus_chan_printf(chan,
869 		    "can not get msg hypercall for chclose(chan%u)\n",
870 		    chan->ch_id);
871 		error = ENXIO;
872 		goto disconnect;
873 	}
874 
875 	req = vmbus_msghc_dataptr(mh);
876 	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHCLOSE;
877 	req->chm_chanid = chan->ch_id;
878 
879 	error = vmbus_msghc_exec_noresult(mh);
880 	vmbus_msghc_put(sc, mh);
881 
882 	if (error) {
883 		vmbus_chan_printf(chan,
884 		    "chclose(chan%u) msg hypercall exec failed: %d\n",
885 		    chan->ch_id, error);
886 		goto disconnect;
887 	}
888 
889 	if (bootverbose)
890 		vmbus_chan_printf(chan, "chan%u closed\n", chan->ch_id);
891 
892 disconnect:
893 	/*
894 	 * Disconnect the TX+RX bufrings from this channel.
895 	 */
896 	if (chan->ch_bufring_gpadl != 0) {
897 		int error1;
898 
899 		error1 = vmbus_chan_gpadl_disconnect(chan,
900 		    chan->ch_bufring_gpadl);
901 		if (error1) {
902 			/*
903 			 * XXX
904 			 * The bufring GPADL is still connected; abandon
905 			 * this bufring, instead of having mysterious
906 			 * crash or trashed data later on.
907 			 */
908 			vmbus_chan_printf(chan, "chan%u bufring GPADL "
909 			    "is still connected after close\n", chan->ch_id);
910 			chan->ch_bufring = NULL;
911 			/*
912 			 * Give caller a hint that the bufring GPADL is
913 			 * still connected.
914 			 */
915 			error = EISCONN;
916 		}
917 		chan->ch_bufring_gpadl = 0;
918 	}
919 
920 	/*
921 	 * Destroy the TX+RX bufrings.
922 	 */
923 	if (chan->ch_bufring != NULL) {
924 		hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring);
925 		chan->ch_bufring = NULL;
926 	}
927 	return (error);
928 }
929 
930 int
931 vmbus_chan_close_direct(struct vmbus_channel *chan)
932 {
933 	int error;
934 
935 #ifdef INVARIANTS
936 	if (VMBUS_CHAN_ISPRIMARY(chan)) {
937 		struct vmbus_channel *subchan;
938 
939 		/*
940 		 * All sub-channels _must_ have been closed, or are _not_
941 		 * opened at all.
942 		 */
943 		mtx_lock(&chan->ch_subchan_lock);
944 		TAILQ_FOREACH(subchan, &chan->ch_subchans, ch_sublink) {
945 			KASSERT(
946 			   (subchan->ch_stflags & VMBUS_CHAN_ST_OPENED) == 0,
947 			   ("chan%u: subchan%u is still opened",
948 			    chan->ch_id, subchan->ch_subidx));
949 		}
950 		mtx_unlock(&chan->ch_subchan_lock);
951 	}
952 #endif
953 
954 	error = vmbus_chan_close_internal(chan);
955 	if (!VMBUS_CHAN_ISPRIMARY(chan)) {
956 		/*
957 		 * This sub-channel is referenced, when it is linked to
958 		 * the primary channel; drop that reference now.
959 		 */
960 		vmbus_chan_detach(chan);
961 	}
962 	return (error);
963 }
964 
965 /*
966  * Caller should make sure that all sub-channels have
967  * been added to 'chan' and all to-be-closed channels
968  * are not being opened.
969  */
970 void
971 vmbus_chan_close(struct vmbus_channel *chan)
972 {
973 	int subchan_cnt;
974 
975 	if (!VMBUS_CHAN_ISPRIMARY(chan)) {
976 		/*
977 		 * Sub-channel is closed when its primary channel
978 		 * is closed; done.
979 		 */
980 		return;
981 	}
982 
983 	/*
984 	 * Close all sub-channels, if any.
985 	 */
986 	subchan_cnt = chan->ch_subchan_cnt;
987 	if (subchan_cnt > 0) {
988 		struct vmbus_channel **subchan;
989 		int i;
990 
991 		subchan = vmbus_subchan_get(chan, subchan_cnt);
992 		for (i = 0; i < subchan_cnt; ++i) {
993 			vmbus_chan_close_internal(subchan[i]);
994 			/*
995 			 * This sub-channel is referenced, when it is
996 			 * linked to the primary channel; drop that
997 			 * reference now.
998 			 */
999 			vmbus_chan_detach(subchan[i]);
1000 		}
1001 		vmbus_subchan_rel(subchan, subchan_cnt);
1002 	}
1003 
1004 	/* Then close the primary channel. */
1005 	vmbus_chan_close_internal(chan);
1006 }
1007 
1008 void
1009 vmbus_chan_intr_drain(struct vmbus_channel *chan)
1010 {
1011 
1012 	taskqueue_drain(chan->ch_tq, &chan->ch_task);
1013 }
1014 
1015 int
1016 vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags,
1017     void *data, int dlen, uint64_t xactid)
1018 {
1019 	struct vmbus_chanpkt pkt;
1020 	int pktlen, pad_pktlen, hlen, error;
1021 	uint64_t pad = 0;
1022 	struct iovec iov[3];
1023 	boolean_t send_evt;
1024 
1025 	hlen = sizeof(pkt);
1026 	pktlen = hlen + dlen;
1027 	pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen);
1028 	KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr),
1029 	    ("invalid packet size %d", pad_pktlen));
1030 
1031 	pkt.cp_hdr.cph_type = type;
1032 	pkt.cp_hdr.cph_flags = flags;
1033 	VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen);
1034 	VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen);
1035 	pkt.cp_hdr.cph_xactid = xactid;
1036 
1037 	iov[0].iov_base = &pkt;
1038 	iov[0].iov_len = hlen;
1039 	iov[1].iov_base = data;
1040 	iov[1].iov_len = dlen;
1041 	iov[2].iov_base = &pad;
1042 	iov[2].iov_len = pad_pktlen - pktlen;
1043 
1044 	error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt);
1045 	if (!error && send_evt)
1046 		vmbus_chan_signal_tx(chan);
1047 	return error;
1048 }
1049 
1050 int
1051 vmbus_chan_send_sglist(struct vmbus_channel *chan,
1052     struct vmbus_gpa sg[], int sglen, void *data, int dlen, uint64_t xactid)
1053 {
1054 	struct vmbus_chanpkt_sglist pkt;
1055 	int pktlen, pad_pktlen, hlen, error;
1056 	struct iovec iov[4];
1057 	boolean_t send_evt;
1058 	uint64_t pad = 0;
1059 
1060 	hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]);
1061 	pktlen = hlen + dlen;
1062 	pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen);
1063 	KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr),
1064 	    ("invalid packet size %d", pad_pktlen));
1065 
1066 	pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA;
1067 	pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC;
1068 	VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen);
1069 	VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen);
1070 	pkt.cp_hdr.cph_xactid = xactid;
1071 	pkt.cp_rsvd = 0;
1072 	pkt.cp_gpa_cnt = sglen;
1073 
1074 	iov[0].iov_base = &pkt;
1075 	iov[0].iov_len = sizeof(pkt);
1076 	iov[1].iov_base = sg;
1077 	iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen;
1078 	iov[2].iov_base = data;
1079 	iov[2].iov_len = dlen;
1080 	iov[3].iov_base = &pad;
1081 	iov[3].iov_len = pad_pktlen - pktlen;
1082 
1083 	error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt);
1084 	if (!error && send_evt)
1085 		vmbus_chan_signal_tx(chan);
1086 	return error;
1087 }
1088 
1089 int
1090 vmbus_chan_send_prplist(struct vmbus_channel *chan,
1091     struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen,
1092     uint64_t xactid)
1093 {
1094 	struct vmbus_chanpkt_prplist pkt;
1095 	int pktlen, pad_pktlen, hlen, error;
1096 	struct iovec iov[4];
1097 	boolean_t send_evt;
1098 	uint64_t pad = 0;
1099 
1100 	hlen = __offsetof(struct vmbus_chanpkt_prplist,
1101 	    cp_range[0].gpa_page[prp_cnt]);
1102 	pktlen = hlen + dlen;
1103 	pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen);
1104 	KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr),
1105 	    ("invalid packet size %d", pad_pktlen));
1106 
1107 	pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA;
1108 	pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC;
1109 	VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_hlen, hlen);
1110 	VMBUS_CHANPKT_SETLEN(pkt.cp_hdr.cph_tlen, pad_pktlen);
1111 	pkt.cp_hdr.cph_xactid = xactid;
1112 	pkt.cp_rsvd = 0;
1113 	pkt.cp_range_cnt = 1;
1114 
1115 	iov[0].iov_base = &pkt;
1116 	iov[0].iov_len = sizeof(pkt);
1117 	iov[1].iov_base = prp;
1118 	iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]);
1119 	iov[2].iov_base = data;
1120 	iov[2].iov_len = dlen;
1121 	iov[3].iov_base = &pad;
1122 	iov[3].iov_len = pad_pktlen - pktlen;
1123 
1124 	error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt);
1125 	if (!error && send_evt)
1126 		vmbus_chan_signal_tx(chan);
1127 	return error;
1128 }
1129 
1130 int
1131 vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0,
1132     uint64_t *xactid)
1133 {
1134 	struct vmbus_chanpkt_hdr pkt;
1135 	int error, dlen, hlen;
1136 
1137 	error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt));
1138 	if (error)
1139 		return (error);
1140 
1141 	if (__predict_false(pkt.cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) {
1142 		vmbus_chan_printf(chan, "invalid hlen %u\n", pkt.cph_hlen);
1143 		/* XXX this channel is dead actually. */
1144 		return (EIO);
1145 	}
1146 	if (__predict_false(pkt.cph_hlen > pkt.cph_tlen)) {
1147 		vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n",
1148 		    pkt.cph_hlen, pkt.cph_tlen);
1149 		/* XXX this channel is dead actually. */
1150 		return (EIO);
1151 	}
1152 
1153 	hlen = VMBUS_CHANPKT_GETLEN(pkt.cph_hlen);
1154 	dlen = VMBUS_CHANPKT_GETLEN(pkt.cph_tlen) - hlen;
1155 
1156 	if (*dlen0 < dlen) {
1157 		/* Return the size of this packet's data. */
1158 		*dlen0 = dlen;
1159 		return (ENOBUFS);
1160 	}
1161 
1162 	*xactid = pkt.cph_xactid;
1163 	*dlen0 = dlen;
1164 
1165 	/* Skip packet header */
1166 	error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen);
1167 	KASSERT(!error, ("vmbus_rxbr_read failed"));
1168 
1169 	return (0);
1170 }
1171 
1172 int
1173 vmbus_chan_recv_pkt(struct vmbus_channel *chan,
1174     struct vmbus_chanpkt_hdr *pkt, int *pktlen0)
1175 {
1176 	int error, pktlen, pkt_hlen;
1177 
1178 	pkt_hlen = sizeof(*pkt);
1179 	error = vmbus_rxbr_peek(&chan->ch_rxbr, pkt, pkt_hlen);
1180 	if (error)
1181 		return (error);
1182 
1183 	if (__predict_false(pkt->cph_hlen < VMBUS_CHANPKT_HLEN_MIN)) {
1184 		vmbus_chan_printf(chan, "invalid hlen %u\n", pkt->cph_hlen);
1185 		/* XXX this channel is dead actually. */
1186 		return (EIO);
1187 	}
1188 	if (__predict_false(pkt->cph_hlen > pkt->cph_tlen)) {
1189 		vmbus_chan_printf(chan, "invalid hlen %u and tlen %u\n",
1190 		    pkt->cph_hlen, pkt->cph_tlen);
1191 		/* XXX this channel is dead actually. */
1192 		return (EIO);
1193 	}
1194 
1195 	pktlen = VMBUS_CHANPKT_GETLEN(pkt->cph_tlen);
1196 	if (*pktlen0 < pktlen) {
1197 		/* Return the size of this packet. */
1198 		*pktlen0 = pktlen;
1199 		return (ENOBUFS);
1200 	}
1201 	*pktlen0 = pktlen;
1202 
1203 	/*
1204 	 * Skip the fixed-size packet header, which has been filled
1205 	 * by the above vmbus_rxbr_peek().
1206 	 */
1207 	error = vmbus_rxbr_read(&chan->ch_rxbr, pkt + 1,
1208 	    pktlen - pkt_hlen, pkt_hlen);
1209 	KASSERT(!error, ("vmbus_rxbr_read failed"));
1210 
1211 	return (0);
1212 }
1213 
1214 static void
1215 vmbus_chan_task(void *xchan, int pending __unused)
1216 {
1217 	struct vmbus_channel *chan = xchan;
1218 	vmbus_chan_callback_t cb = chan->ch_cb;
1219 	void *cbarg = chan->ch_cbarg;
1220 
1221 	KASSERT(chan->ch_poll_intvl == 0,
1222 	    ("chan%u: interrupted in polling mode", chan->ch_id));
1223 
1224 	/*
1225 	 * Optimize host to guest signaling by ensuring:
1226 	 * 1. While reading the channel, we disable interrupts from
1227 	 *    host.
1228 	 * 2. Ensure that we process all posted messages from the host
1229 	 *    before returning from this callback.
1230 	 * 3. Once we return, enable signaling from the host. Once this
1231 	 *    state is set we check to see if additional packets are
1232 	 *    available to read. In this case we repeat the process.
1233 	 *
1234 	 * NOTE: Interrupt has been disabled in the ISR.
1235 	 */
1236 	for (;;) {
1237 		uint32_t left;
1238 
1239 		cb(chan, cbarg);
1240 
1241 		left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr);
1242 		if (left == 0) {
1243 			/* No more data in RX bufring; done */
1244 			break;
1245 		}
1246 		vmbus_rxbr_intr_mask(&chan->ch_rxbr);
1247 	}
1248 }
1249 
1250 static void
1251 vmbus_chan_task_nobatch(void *xchan, int pending __unused)
1252 {
1253 	struct vmbus_channel *chan = xchan;
1254 
1255 	KASSERT(chan->ch_poll_intvl == 0,
1256 	    ("chan%u: interrupted in polling mode", chan->ch_id));
1257 	chan->ch_cb(chan, chan->ch_cbarg);
1258 }
1259 
1260 static void
1261 vmbus_chan_poll_timeout(void *xchan)
1262 {
1263 	struct vmbus_channel *chan = xchan;
1264 
1265 	KASSERT(chan->ch_poll_intvl != 0,
1266 	    ("chan%u: polling timeout in interrupt mode", chan->ch_id));
1267 	taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task);
1268 }
1269 
1270 static void
1271 vmbus_chan_poll_task(void *xchan, int pending __unused)
1272 {
1273 	struct vmbus_channel *chan = xchan;
1274 
1275 	KASSERT(chan->ch_poll_intvl != 0,
1276 	    ("chan%u: polling in interrupt mode", chan->ch_id));
1277 	callout_reset_sbt_curcpu(&chan->ch_poll_timeo, chan->ch_poll_intvl, 0,
1278 	    vmbus_chan_poll_timeout, chan, chan->ch_poll_flags);
1279 	chan->ch_cb(chan, chan->ch_cbarg);
1280 }
1281 
1282 static void
1283 vmbus_chan_pollcfg_task(void *xarg, int pending __unused)
1284 {
1285 	const struct vmbus_chan_pollarg *arg = xarg;
1286 	struct vmbus_channel *chan = arg->poll_chan;
1287 	sbintime_t intvl;
1288 	int poll_flags;
1289 
1290 	/*
1291 	 * Save polling interval.
1292 	 */
1293 	intvl = SBT_1S / arg->poll_hz;
1294 	if (intvl == 0)
1295 		intvl = 1;
1296 	if (intvl == chan->ch_poll_intvl) {
1297 		/* Nothing changes; done */
1298 		return;
1299 	}
1300 	chan->ch_poll_intvl = intvl;
1301 
1302 	/* Adjust callout flags. */
1303 	poll_flags = C_DIRECT_EXEC;
1304 	if (arg->poll_hz <= hz)
1305 		poll_flags |= C_HARDCLOCK;
1306 	chan->ch_poll_flags = poll_flags;
1307 
1308 	/*
1309 	 * Disconnect this channel from the channel map to make sure that
1310 	 * the RX bufring interrupt enabling bit can not be touched, and
1311 	 * ISR can not enqueue this channel task anymore.  THEN, disable
1312 	 * interrupt from the RX bufring (TX bufring does not generate
1313 	 * interrupt to VM).
1314 	 *
1315 	 * NOTE: order is critical.
1316 	 */
1317 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = NULL;
1318 	__compiler_membar();
1319 	vmbus_rxbr_intr_mask(&chan->ch_rxbr);
1320 
1321 	/*
1322 	 * NOTE:
1323 	 * At this point, this channel task will not be enqueued by
1324 	 * the ISR anymore, time to cancel the pending one.
1325 	 */
1326 	taskqueue_cancel(chan->ch_tq, &chan->ch_task, NULL);
1327 
1328 	/* Kick start! */
1329 	taskqueue_enqueue(chan->ch_tq, &chan->ch_poll_task);
1330 }
1331 
1332 static bool
1333 vmbus_chan_poll_cancel_intq(struct vmbus_channel *chan)
1334 {
1335 
1336 	if (chan->ch_poll_intvl == 0) {
1337 		/* Not enabled. */
1338 		return (false);
1339 	}
1340 
1341 	/*
1342 	 * Stop polling callout, so that channel polling task
1343 	 * will not be enqueued anymore.
1344 	 */
1345 	callout_drain(&chan->ch_poll_timeo);
1346 
1347 	/*
1348 	 * Disable polling by resetting polling interval.
1349 	 *
1350 	 * NOTE:
1351 	 * The polling interval resetting MUST be conducted
1352 	 * after the callout is drained; mainly to keep the
1353 	 * proper assertion in place.
1354 	 */
1355 	chan->ch_poll_intvl = 0;
1356 
1357 	/*
1358 	 * NOTE:
1359 	 * At this point, this channel polling task will not be
1360 	 * enqueued by the callout anymore, time to cancel the
1361 	 * pending one.
1362 	 */
1363 	taskqueue_cancel(chan->ch_tq, &chan->ch_poll_task, NULL);
1364 
1365 	/* Polling was enabled. */
1366 	return (true);
1367 }
1368 
1369 static void
1370 vmbus_chan_polldis_task(void *xchan, int pending __unused)
1371 {
1372 	struct vmbus_channel *chan = xchan;
1373 
1374 	if (!vmbus_chan_poll_cancel_intq(chan)) {
1375 		/* Already disabled; done. */
1376 		return;
1377 	}
1378 
1379 	/*
1380 	 * Plug this channel back to the channel map and unmask
1381 	 * the RX bufring interrupt.
1382 	 */
1383 	chan->ch_vmbus->vmbus_chmap[chan->ch_id] = chan;
1384 	__compiler_membar();
1385 	vmbus_rxbr_intr_unmask(&chan->ch_rxbr);
1386 
1387 	/*
1388 	 * Kick start the interrupt task, just in case unmasking
1389 	 * interrupt races ISR.
1390 	 */
1391 	taskqueue_enqueue(chan->ch_tq, &chan->ch_task);
1392 }
1393 
1394 static __inline void
1395 vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags,
1396     int flag_cnt)
1397 {
1398 	int f;
1399 
1400 	for (f = 0; f < flag_cnt; ++f) {
1401 		uint32_t chid_base;
1402 		u_long flags;
1403 		int chid_ofs;
1404 
1405 		if (event_flags[f] == 0)
1406 			continue;
1407 
1408 		flags = atomic_swap_long(&event_flags[f], 0);
1409 		chid_base = f << VMBUS_EVTFLAG_SHIFT;
1410 
1411 		while ((chid_ofs = ffsl(flags)) != 0) {
1412 			struct vmbus_channel *chan;
1413 
1414 			--chid_ofs; /* NOTE: ffsl is 1-based */
1415 			flags &= ~(1UL << chid_ofs);
1416 
1417 			chan = sc->vmbus_chmap[chid_base + chid_ofs];
1418 			if (__predict_false(chan == NULL)) {
1419 				/* Channel is closed. */
1420 				continue;
1421 			}
1422 			__compiler_membar();
1423 
1424 			if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD)
1425 				vmbus_rxbr_intr_mask(&chan->ch_rxbr);
1426 			taskqueue_enqueue(chan->ch_tq, &chan->ch_task);
1427 		}
1428 	}
1429 }
1430 
1431 void
1432 vmbus_event_proc(struct vmbus_softc *sc, int cpu)
1433 {
1434 	struct vmbus_evtflags *eventf;
1435 
1436 	/*
1437 	 * On Host with Win8 or above, the event page can be checked directly
1438 	 * to get the id of the channel that has the pending interrupt.
1439 	 */
1440 	eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE;
1441 	vmbus_event_flags_proc(sc, eventf->evt_flags,
1442 	    VMBUS_PCPU_GET(sc, event_flags_cnt, cpu));
1443 }
1444 
1445 void
1446 vmbus_event_proc_compat(struct vmbus_softc *sc, int cpu)
1447 {
1448 	struct vmbus_evtflags *eventf;
1449 
1450 	eventf = VMBUS_PCPU_GET(sc, event_flags, cpu) + VMBUS_SINT_MESSAGE;
1451 	if (atomic_testandclear_long(&eventf->evt_flags[0], 0)) {
1452 		vmbus_event_flags_proc(sc, sc->vmbus_rx_evtflags,
1453 		    VMBUS_CHAN_MAX_COMPAT >> VMBUS_EVTFLAG_SHIFT);
1454 	}
1455 }
1456 
1457 static void
1458 vmbus_chan_update_evtflagcnt(struct vmbus_softc *sc,
1459     const struct vmbus_channel *chan)
1460 {
1461 	volatile int *flag_cnt_ptr;
1462 	int flag_cnt;
1463 
1464 	flag_cnt = (chan->ch_id / VMBUS_EVTFLAG_LEN) + 1;
1465 	flag_cnt_ptr = VMBUS_PCPU_PTR(sc, event_flags_cnt, chan->ch_cpuid);
1466 
1467 	for (;;) {
1468 		int old_flag_cnt;
1469 
1470 		old_flag_cnt = *flag_cnt_ptr;
1471 		if (old_flag_cnt >= flag_cnt)
1472 			break;
1473 		if (atomic_cmpset_int(flag_cnt_ptr, old_flag_cnt, flag_cnt)) {
1474 			if (bootverbose) {
1475 				vmbus_chan_printf(chan,
1476 				    "chan%u update cpu%d flag_cnt to %d\n",
1477 				    chan->ch_id, chan->ch_cpuid, flag_cnt);
1478 			}
1479 			break;
1480 		}
1481 	}
1482 }
1483 
1484 static struct vmbus_channel *
1485 vmbus_chan_alloc(struct vmbus_softc *sc)
1486 {
1487 	struct vmbus_channel *chan;
1488 
1489 	chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO);
1490 
1491 	chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev),
1492 	    HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param),
1493 	    &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
1494 	if (chan->ch_monprm == NULL) {
1495 		device_printf(sc->vmbus_dev, "monprm alloc failed\n");
1496 		free(chan, M_DEVBUF);
1497 		return NULL;
1498 	}
1499 
1500 	chan->ch_refs = 1;
1501 	chan->ch_vmbus = sc;
1502 	mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF);
1503 	sx_init(&chan->ch_orphan_lock, "vmbus chorphan");
1504 	TAILQ_INIT(&chan->ch_subchans);
1505 	vmbus_rxbr_init(&chan->ch_rxbr);
1506 	vmbus_txbr_init(&chan->ch_txbr);
1507 
1508 	TASK_INIT(&chan->ch_poll_task, 0, vmbus_chan_poll_task, chan);
1509 	callout_init(&chan->ch_poll_timeo, 1);
1510 
1511 	return chan;
1512 }
1513 
1514 static void
1515 vmbus_chan_free(struct vmbus_channel *chan)
1516 {
1517 
1518 	KASSERT(TAILQ_EMPTY(&chan->ch_subchans) && chan->ch_subchan_cnt == 0,
1519 	    ("still owns sub-channels"));
1520 	KASSERT((chan->ch_stflags &
1521 	    (VMBUS_CHAN_ST_OPENED |
1522 	     VMBUS_CHAN_ST_ONPRIL |
1523 	     VMBUS_CHAN_ST_ONSUBL |
1524 	     VMBUS_CHAN_ST_ONLIST)) == 0, ("free busy channel"));
1525 	KASSERT(chan->ch_orphan_xact == NULL,
1526 	    ("still has orphan xact installed"));
1527 	KASSERT(chan->ch_refs == 0, ("chan%u: invalid refcnt %d",
1528 	    chan->ch_id, chan->ch_refs));
1529 	KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated",
1530 	    chan->ch_id));
1531 
1532 	hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm);
1533 	mtx_destroy(&chan->ch_subchan_lock);
1534 	sx_destroy(&chan->ch_orphan_lock);
1535 	vmbus_rxbr_deinit(&chan->ch_rxbr);
1536 	vmbus_txbr_deinit(&chan->ch_txbr);
1537 	free(chan, M_DEVBUF);
1538 }
1539 
1540 static int
1541 vmbus_chan_add(struct vmbus_channel *newchan)
1542 {
1543 	struct vmbus_softc *sc = newchan->ch_vmbus;
1544 	struct vmbus_channel *prichan;
1545 
1546 	if (newchan->ch_id == 0) {
1547 		/*
1548 		 * XXX
1549 		 * Chan0 will neither be processed nor should be offered;
1550 		 * skip it.
1551 		 */
1552 		device_printf(sc->vmbus_dev, "got chan0 offer, discard\n");
1553 		return EINVAL;
1554 	} else if (newchan->ch_id >= VMBUS_CHAN_MAX) {
1555 		device_printf(sc->vmbus_dev, "invalid chan%u offer\n",
1556 		    newchan->ch_id);
1557 		return EINVAL;
1558 	}
1559 
1560 	mtx_lock(&sc->vmbus_prichan_lock);
1561 	TAILQ_FOREACH(prichan, &sc->vmbus_prichans, ch_prilink) {
1562 		/*
1563 		 * Sub-channel will have the same type GUID and instance
1564 		 * GUID as its primary channel.
1565 		 */
1566 		if (memcmp(&prichan->ch_guid_type, &newchan->ch_guid_type,
1567 		    sizeof(struct hyperv_guid)) == 0 &&
1568 		    memcmp(&prichan->ch_guid_inst, &newchan->ch_guid_inst,
1569 		    sizeof(struct hyperv_guid)) == 0)
1570 			break;
1571 	}
1572 	if (VMBUS_CHAN_ISPRIMARY(newchan)) {
1573 		if (prichan == NULL) {
1574 			/* Install the new primary channel */
1575 			vmbus_chan_ins_prilist(sc, newchan);
1576 			mtx_unlock(&sc->vmbus_prichan_lock);
1577 			goto done;
1578 		} else {
1579 			mtx_unlock(&sc->vmbus_prichan_lock);
1580 			device_printf(sc->vmbus_dev,
1581 			    "duplicated primary chan%u\n", newchan->ch_id);
1582 			return EINVAL;
1583 		}
1584 	} else { /* Sub-channel */
1585 		if (prichan == NULL) {
1586 			mtx_unlock(&sc->vmbus_prichan_lock);
1587 			device_printf(sc->vmbus_dev,
1588 			    "no primary chan for chan%u\n", newchan->ch_id);
1589 			return EINVAL;
1590 		}
1591 		/*
1592 		 * Found the primary channel for this sub-channel and
1593 		 * move on.
1594 		 *
1595 		 * XXX refcnt prichan
1596 		 */
1597 	}
1598 	mtx_unlock(&sc->vmbus_prichan_lock);
1599 
1600 	/*
1601 	 * This is a sub-channel; link it with the primary channel.
1602 	 */
1603 	KASSERT(!VMBUS_CHAN_ISPRIMARY(newchan),
1604 	    ("new channel is not sub-channel"));
1605 	KASSERT(prichan != NULL, ("no primary channel"));
1606 
1607 	/*
1608 	 * Reference count this sub-channel; it will be dereferenced
1609 	 * when this sub-channel is closed.
1610 	 */
1611 	KASSERT(newchan->ch_refs == 1, ("chan%u: invalid refcnt %d",
1612 	    newchan->ch_id, newchan->ch_refs));
1613 	atomic_add_int(&newchan->ch_refs, 1);
1614 
1615 	newchan->ch_prichan = prichan;
1616 	newchan->ch_dev = prichan->ch_dev;
1617 
1618 	mtx_lock(&prichan->ch_subchan_lock);
1619 	vmbus_chan_ins_sublist(prichan, newchan);
1620 	mtx_unlock(&prichan->ch_subchan_lock);
1621 	/*
1622 	 * Notify anyone that is interested in this sub-channel,
1623 	 * after this sub-channel is setup.
1624 	 */
1625 	wakeup(prichan);
1626 done:
1627 	/*
1628 	 * Hook this channel up for later revocation.
1629 	 */
1630 	mtx_lock(&sc->vmbus_chan_lock);
1631 	vmbus_chan_ins_list(sc, newchan);
1632 	mtx_unlock(&sc->vmbus_chan_lock);
1633 
1634 	if (bootverbose) {
1635 		vmbus_chan_printf(newchan, "chan%u subidx%u offer\n",
1636 		    newchan->ch_id, newchan->ch_subidx);
1637 	}
1638 
1639 	/* Select default cpu for this channel. */
1640 	vmbus_chan_cpu_default(newchan);
1641 
1642 	return 0;
1643 }
1644 
1645 void
1646 vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu)
1647 {
1648 	KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu %d", cpu));
1649 
1650 	if (chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WS2008 ||
1651 	    chan->ch_vmbus->vmbus_version == VMBUS_VERSION_WIN7) {
1652 		/* Only cpu0 is supported */
1653 		cpu = 0;
1654 	}
1655 
1656 	chan->ch_cpuid = cpu;
1657 	chan->ch_vcpuid = VMBUS_PCPU_GET(chan->ch_vmbus, vcpuid, cpu);
1658 
1659 	if (bootverbose) {
1660 		vmbus_chan_printf(chan,
1661 		    "chan%u assigned to cpu%u [vcpu%u]\n",
1662 		    chan->ch_id, chan->ch_cpuid, chan->ch_vcpuid);
1663 	}
1664 }
1665 
1666 void
1667 vmbus_chan_cpu_rr(struct vmbus_channel *chan)
1668 {
1669 	static uint32_t vmbus_chan_nextcpu;
1670 	int cpu;
1671 
1672 	cpu = atomic_fetchadd_int(&vmbus_chan_nextcpu, 1) % mp_ncpus;
1673 	vmbus_chan_cpu_set(chan, cpu);
1674 }
1675 
1676 static void
1677 vmbus_chan_cpu_default(struct vmbus_channel *chan)
1678 {
1679 	/*
1680 	 * By default, pin the channel to cpu0.  Devices having
1681 	 * special channel-cpu mapping requirement should call
1682 	 * vmbus_chan_cpu_{set,rr}().
1683 	 */
1684 	vmbus_chan_cpu_set(chan, 0);
1685 }
1686 
1687 static void
1688 vmbus_chan_msgproc_choffer(struct vmbus_softc *sc,
1689     const struct vmbus_message *msg)
1690 {
1691 	const struct vmbus_chanmsg_choffer *offer;
1692 	struct vmbus_channel *chan;
1693 	task_fn_t *detach_fn, *attach_fn;
1694 	int error;
1695 
1696 	offer = (const struct vmbus_chanmsg_choffer *)msg->msg_data;
1697 
1698 	chan = vmbus_chan_alloc(sc);
1699 	if (chan == NULL) {
1700 		device_printf(sc->vmbus_dev, "allocate chan%u failed\n",
1701 		    offer->chm_chanid);
1702 		return;
1703 	}
1704 
1705 	chan->ch_id = offer->chm_chanid;
1706 	chan->ch_subidx = offer->chm_subidx;
1707 	chan->ch_guid_type = offer->chm_chtype;
1708 	chan->ch_guid_inst = offer->chm_chinst;
1709 
1710 	/* Batch reading is on by default */
1711 	chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD;
1712 
1713 	chan->ch_monprm->mp_connid = VMBUS_CONNID_EVENT;
1714 	if (sc->vmbus_version != VMBUS_VERSION_WS2008)
1715 		chan->ch_monprm->mp_connid = offer->chm_connid;
1716 
1717 	if (offer->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) {
1718 		int trig_idx;
1719 
1720 		/*
1721 		 * Setup MNF stuffs.
1722 		 */
1723 		chan->ch_txflags |= VMBUS_CHAN_TXF_HASMNF;
1724 
1725 		trig_idx = offer->chm_montrig / VMBUS_MONTRIG_LEN;
1726 		if (trig_idx >= VMBUS_MONTRIGS_MAX)
1727 			panic("invalid monitor trigger %u", offer->chm_montrig);
1728 		chan->ch_montrig =
1729 		    &sc->vmbus_mnf2->mnf_trigs[trig_idx].mt_pending;
1730 
1731 		chan->ch_montrig_mask =
1732 		    1 << (offer->chm_montrig % VMBUS_MONTRIG_LEN);
1733 	}
1734 
1735 	/*
1736 	 * Setup event flag.
1737 	 */
1738 	chan->ch_evtflag =
1739 	    &sc->vmbus_tx_evtflags[chan->ch_id >> VMBUS_EVTFLAG_SHIFT];
1740 	chan->ch_evtflag_mask = 1UL << (chan->ch_id & VMBUS_EVTFLAG_MASK);
1741 
1742 	/*
1743 	 * Setup attach and detach tasks.
1744 	 */
1745 	if (VMBUS_CHAN_ISPRIMARY(chan)) {
1746 		chan->ch_mgmt_tq = sc->vmbus_devtq;
1747 		attach_fn = vmbus_prichan_attach_task;
1748 		detach_fn = vmbus_prichan_detach_task;
1749 	} else {
1750 		chan->ch_mgmt_tq = sc->vmbus_subchtq;
1751 		attach_fn = vmbus_subchan_attach_task;
1752 		detach_fn = vmbus_subchan_detach_task;
1753 	}
1754 	TASK_INIT(&chan->ch_attach_task, 0, attach_fn, chan);
1755 	TASK_INIT(&chan->ch_detach_task, 0, detach_fn, chan);
1756 
1757 	error = vmbus_chan_add(chan);
1758 	if (error) {
1759 		device_printf(sc->vmbus_dev, "add chan%u failed: %d\n",
1760 		    chan->ch_id, error);
1761 		atomic_subtract_int(&chan->ch_refs, 1);
1762 		vmbus_chan_free(chan);
1763 		return;
1764 	}
1765 	taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_attach_task);
1766 }
1767 
1768 static void
1769 vmbus_chan_msgproc_chrescind(struct vmbus_softc *sc,
1770     const struct vmbus_message *msg)
1771 {
1772 	const struct vmbus_chanmsg_chrescind *note;
1773 	struct vmbus_channel *chan;
1774 
1775 	note = (const struct vmbus_chanmsg_chrescind *)msg->msg_data;
1776 	if (note->chm_chanid > VMBUS_CHAN_MAX) {
1777 		device_printf(sc->vmbus_dev, "invalid revoked chan%u\n",
1778 		    note->chm_chanid);
1779 		return;
1780 	}
1781 
1782 	/*
1783 	 * Find and remove the target channel from the channel list.
1784 	 */
1785 	mtx_lock(&sc->vmbus_chan_lock);
1786 	TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) {
1787 		if (chan->ch_id == note->chm_chanid)
1788 			break;
1789 	}
1790 	if (chan == NULL) {
1791 		mtx_unlock(&sc->vmbus_chan_lock);
1792 		device_printf(sc->vmbus_dev, "chan%u is not offered\n",
1793 		    note->chm_chanid);
1794 		return;
1795 	}
1796 	vmbus_chan_rem_list(sc, chan);
1797 	mtx_unlock(&sc->vmbus_chan_lock);
1798 
1799 	if (VMBUS_CHAN_ISPRIMARY(chan)) {
1800 		/*
1801 		 * The target channel is a primary channel; remove the
1802 		 * target channel from the primary channel list now,
1803 		 * instead of later, so that it will not be found by
1804 		 * other sub-channel offers, which are processed in
1805 		 * this thread.
1806 		 */
1807 		mtx_lock(&sc->vmbus_prichan_lock);
1808 		vmbus_chan_rem_prilist(sc, chan);
1809 		mtx_unlock(&sc->vmbus_prichan_lock);
1810 	}
1811 
1812 	/*
1813 	 * NOTE:
1814 	 * The following processing order is critical:
1815 	 * Set the REVOKED state flag before orphaning the installed xact.
1816 	 */
1817 
1818 	if (atomic_testandset_int(&chan->ch_stflags,
1819 	    VMBUS_CHAN_ST_REVOKED_SHIFT))
1820 		panic("channel has already been revoked");
1821 
1822 	sx_xlock(&chan->ch_orphan_lock);
1823 	if (chan->ch_orphan_xact != NULL)
1824 		vmbus_xact_ctx_orphan(chan->ch_orphan_xact);
1825 	sx_xunlock(&chan->ch_orphan_lock);
1826 
1827 	if (bootverbose)
1828 		vmbus_chan_printf(chan, "chan%u revoked\n", note->chm_chanid);
1829 	vmbus_chan_detach(chan);
1830 }
1831 
1832 static int
1833 vmbus_chan_release(struct vmbus_channel *chan)
1834 {
1835 	struct vmbus_softc *sc = chan->ch_vmbus;
1836 	struct vmbus_chanmsg_chfree *req;
1837 	struct vmbus_msghc *mh;
1838 	int error;
1839 
1840 	mh = vmbus_msghc_get(sc, sizeof(*req));
1841 	if (mh == NULL) {
1842 		vmbus_chan_printf(chan,
1843 		    "can not get msg hypercall for chfree(chan%u)\n",
1844 		    chan->ch_id);
1845 		return (ENXIO);
1846 	}
1847 
1848 	req = vmbus_msghc_dataptr(mh);
1849 	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE;
1850 	req->chm_chanid = chan->ch_id;
1851 
1852 	error = vmbus_msghc_exec_noresult(mh);
1853 	vmbus_msghc_put(sc, mh);
1854 
1855 	if (error) {
1856 		vmbus_chan_printf(chan,
1857 		    "chfree(chan%u) msg hypercall exec failed: %d\n",
1858 		    chan->ch_id, error);
1859 	} else {
1860 		if (bootverbose)
1861 			vmbus_chan_printf(chan, "chan%u freed\n", chan->ch_id);
1862 	}
1863 	return (error);
1864 }
1865 
1866 static void
1867 vmbus_prichan_detach_task(void *xchan, int pending __unused)
1868 {
1869 	struct vmbus_channel *chan = xchan;
1870 
1871 	KASSERT(VMBUS_CHAN_ISPRIMARY(chan),
1872 	    ("chan%u is not primary channel", chan->ch_id));
1873 
1874 	/* Delete and detach the device associated with this channel. */
1875 	vmbus_delete_child(chan);
1876 
1877 	/* Release this channel (back to vmbus). */
1878 	vmbus_chan_release(chan);
1879 
1880 	/* Free this channel's resource. */
1881 	vmbus_chan_free(chan);
1882 }
1883 
1884 static void
1885 vmbus_subchan_detach_task(void *xchan, int pending __unused)
1886 {
1887 	struct vmbus_channel *chan = xchan;
1888 	struct vmbus_channel *pri_chan = chan->ch_prichan;
1889 
1890 	KASSERT(!VMBUS_CHAN_ISPRIMARY(chan),
1891 	    ("chan%u is primary channel", chan->ch_id));
1892 
1893 	/* Release this channel (back to vmbus). */
1894 	vmbus_chan_release(chan);
1895 
1896 	/* Unlink from its primary channel's sub-channel list. */
1897 	mtx_lock(&pri_chan->ch_subchan_lock);
1898 	vmbus_chan_rem_sublist(pri_chan, chan);
1899 	mtx_unlock(&pri_chan->ch_subchan_lock);
1900 	/* Notify anyone that is waiting for this sub-channel to vanish. */
1901 	wakeup(pri_chan);
1902 
1903 	/* Free this channel's resource. */
1904 	vmbus_chan_free(chan);
1905 }
1906 
1907 static void
1908 vmbus_prichan_attach_task(void *xchan, int pending __unused)
1909 {
1910 
1911 	/*
1912 	 * Add device for this primary channel.
1913 	 */
1914 	vmbus_add_child(xchan);
1915 }
1916 
1917 static void
1918 vmbus_subchan_attach_task(void *xchan __unused, int pending __unused)
1919 {
1920 
1921 	/* Nothing */
1922 }
1923 
1924 void
1925 vmbus_chan_destroy_all(struct vmbus_softc *sc)
1926 {
1927 
1928 	/*
1929 	 * Detach all devices and destroy the corresponding primary
1930 	 * channels.
1931 	 */
1932 	for (;;) {
1933 		struct vmbus_channel *chan;
1934 
1935 		mtx_lock(&sc->vmbus_chan_lock);
1936 		TAILQ_FOREACH(chan, &sc->vmbus_chans, ch_link) {
1937 			if (VMBUS_CHAN_ISPRIMARY(chan))
1938 				break;
1939 		}
1940 		if (chan == NULL) {
1941 			/* No more primary channels; done. */
1942 			mtx_unlock(&sc->vmbus_chan_lock);
1943 			break;
1944 		}
1945 		vmbus_chan_rem_list(sc, chan);
1946 		mtx_unlock(&sc->vmbus_chan_lock);
1947 
1948 		mtx_lock(&sc->vmbus_prichan_lock);
1949 		vmbus_chan_rem_prilist(sc, chan);
1950 		mtx_unlock(&sc->vmbus_prichan_lock);
1951 
1952 		taskqueue_enqueue(chan->ch_mgmt_tq, &chan->ch_detach_task);
1953 	}
1954 }
1955 
1956 struct vmbus_channel **
1957 vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt)
1958 {
1959 	struct vmbus_channel **ret, *chan;
1960 	int i;
1961 
1962 	KASSERT(subchan_cnt > 0, ("invalid sub-channel count %d", subchan_cnt));
1963 
1964 	ret = malloc(subchan_cnt * sizeof(struct vmbus_channel *), M_TEMP,
1965 	    M_WAITOK);
1966 
1967 	mtx_lock(&pri_chan->ch_subchan_lock);
1968 
1969 	while (pri_chan->ch_subchan_cnt < subchan_cnt)
1970 		mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "subch", 0);
1971 
1972 	i = 0;
1973 	TAILQ_FOREACH(chan, &pri_chan->ch_subchans, ch_sublink) {
1974 		/* TODO: refcnt chan */
1975 		ret[i] = chan;
1976 
1977 		++i;
1978 		if (i == subchan_cnt)
1979 			break;
1980 	}
1981 	KASSERT(i == subchan_cnt, ("invalid subchan count %d, should be %d",
1982 	    pri_chan->ch_subchan_cnt, subchan_cnt));
1983 
1984 	mtx_unlock(&pri_chan->ch_subchan_lock);
1985 
1986 	return ret;
1987 }
1988 
1989 void
1990 vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt __unused)
1991 {
1992 
1993 	free(subchan, M_TEMP);
1994 }
1995 
1996 void
1997 vmbus_subchan_drain(struct vmbus_channel *pri_chan)
1998 {
1999 	mtx_lock(&pri_chan->ch_subchan_lock);
2000 	while (pri_chan->ch_subchan_cnt > 0)
2001 		mtx_sleep(pri_chan, &pri_chan->ch_subchan_lock, 0, "dsubch", 0);
2002 	mtx_unlock(&pri_chan->ch_subchan_lock);
2003 }
2004 
2005 void
2006 vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg)
2007 {
2008 	vmbus_chanmsg_proc_t msg_proc;
2009 	uint32_t msg_type;
2010 
2011 	msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type;
2012 	KASSERT(msg_type < VMBUS_CHANMSG_TYPE_MAX,
2013 	    ("invalid message type %u", msg_type));
2014 
2015 	msg_proc = vmbus_chan_msgprocs[msg_type];
2016 	if (msg_proc != NULL)
2017 		msg_proc(sc, msg);
2018 }
2019 
2020 void
2021 vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on)
2022 {
2023 	if (!on)
2024 		chan->ch_flags &= ~VMBUS_CHAN_FLAG_BATCHREAD;
2025 	else
2026 		chan->ch_flags |= VMBUS_CHAN_FLAG_BATCHREAD;
2027 }
2028 
2029 uint32_t
2030 vmbus_chan_id(const struct vmbus_channel *chan)
2031 {
2032 	return chan->ch_id;
2033 }
2034 
2035 uint32_t
2036 vmbus_chan_subidx(const struct vmbus_channel *chan)
2037 {
2038 	return chan->ch_subidx;
2039 }
2040 
2041 bool
2042 vmbus_chan_is_primary(const struct vmbus_channel *chan)
2043 {
2044 	if (VMBUS_CHAN_ISPRIMARY(chan))
2045 		return true;
2046 	else
2047 		return false;
2048 }
2049 
2050 const struct hyperv_guid *
2051 vmbus_chan_guid_inst(const struct vmbus_channel *chan)
2052 {
2053 	return &chan->ch_guid_inst;
2054 }
2055 
2056 int
2057 vmbus_chan_prplist_nelem(int br_size, int prpcnt_max, int dlen_max)
2058 {
2059 	int elem_size;
2060 
2061 	elem_size = __offsetof(struct vmbus_chanpkt_prplist,
2062 	    cp_range[0].gpa_page[prpcnt_max]);
2063 	elem_size += dlen_max;
2064 	elem_size = VMBUS_CHANPKT_TOTLEN(elem_size);
2065 
2066 	return (vmbus_br_nelem(br_size, elem_size));
2067 }
2068 
2069 bool
2070 vmbus_chan_tx_empty(const struct vmbus_channel *chan)
2071 {
2072 
2073 	return (vmbus_txbr_empty(&chan->ch_txbr));
2074 }
2075 
2076 bool
2077 vmbus_chan_rx_empty(const struct vmbus_channel *chan)
2078 {
2079 
2080 	return (vmbus_rxbr_empty(&chan->ch_rxbr));
2081 }
2082 
2083 static int
2084 vmbus_chan_printf(const struct vmbus_channel *chan, const char *fmt, ...)
2085 {
2086 	va_list ap;
2087 	device_t dev;
2088 	int retval;
2089 
2090 	if (chan->ch_dev == NULL || !device_is_alive(chan->ch_dev))
2091 		dev = chan->ch_vmbus->vmbus_dev;
2092 	else
2093 		dev = chan->ch_dev;
2094 
2095 	retval = device_print_prettyname(dev);
2096 	va_start(ap, fmt);
2097 	retval += vprintf(fmt, ap);
2098 	va_end(ap);
2099 
2100 	return (retval);
2101 }
2102 
2103 void
2104 vmbus_chan_run_task(struct vmbus_channel *chan, struct task *task)
2105 {
2106 
2107 	taskqueue_enqueue(chan->ch_tq, task);
2108 	taskqueue_drain(chan->ch_tq, task);
2109 }
2110 
2111 struct taskqueue *
2112 vmbus_chan_mgmt_tq(const struct vmbus_channel *chan)
2113 {
2114 
2115 	return (chan->ch_mgmt_tq);
2116 }
2117 
2118 bool
2119 vmbus_chan_is_revoked(const struct vmbus_channel *chan)
2120 {
2121 
2122 	if (chan->ch_stflags & VMBUS_CHAN_ST_REVOKED)
2123 		return (true);
2124 	return (false);
2125 }
2126 
2127 void
2128 vmbus_chan_set_orphan(struct vmbus_channel *chan, struct vmbus_xact_ctx *xact)
2129 {
2130 
2131 	sx_xlock(&chan->ch_orphan_lock);
2132 	chan->ch_orphan_xact = xact;
2133 	sx_xunlock(&chan->ch_orphan_lock);
2134 }
2135 
2136 void
2137 vmbus_chan_unset_orphan(struct vmbus_channel *chan)
2138 {
2139 
2140 	sx_xlock(&chan->ch_orphan_lock);
2141 	chan->ch_orphan_xact = NULL;
2142 	sx_xunlock(&chan->ch_orphan_lock);
2143 }
2144 
2145 const void *
2146 vmbus_chan_xact_wait(const struct vmbus_channel *chan,
2147     struct vmbus_xact *xact, size_t *resp_len, bool can_sleep)
2148 {
2149 	const void *ret;
2150 
2151 	if (can_sleep)
2152 		ret = vmbus_xact_wait(xact, resp_len);
2153 	else
2154 		ret = vmbus_xact_busywait(xact, resp_len);
2155 	if (vmbus_chan_is_revoked(chan)) {
2156 		/*
2157 		 * This xact probably is interrupted, and the
2158 		 * interruption can race the reply reception,
2159 		 * so we have to make sure that there are nothing
2160 		 * left on the RX bufring, i.e. this xact will
2161 		 * not be touched, once this function returns.
2162 		 *
2163 		 * Since the hypervisor will not put more data
2164 		 * onto the RX bufring once the channel is revoked,
2165 		 * the following loop will be terminated, once all
2166 		 * data are drained by the driver's channel
2167 		 * callback.
2168 		 */
2169 		while (!vmbus_chan_rx_empty(chan)) {
2170 			if (can_sleep)
2171 				pause("chxact", 1);
2172 			else
2173 				DELAY(1000);
2174 		}
2175 	}
2176 	return (ret);
2177 }
2178 
2179 void
2180 vmbus_chan_poll_enable(struct vmbus_channel *chan, u_int pollhz)
2181 {
2182 	struct vmbus_chan_pollarg arg;
2183 	struct task poll_cfg;
2184 
2185 	KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD,
2186 	    ("enable polling on non-batch chan%u", chan->ch_id));
2187 	KASSERT(pollhz >= VMBUS_CHAN_POLLHZ_MIN &&
2188 	    pollhz <= VMBUS_CHAN_POLLHZ_MAX, ("invalid pollhz %u", pollhz));
2189 
2190 	arg.poll_chan = chan;
2191 	arg.poll_hz = pollhz;
2192 	TASK_INIT(&poll_cfg, 0, vmbus_chan_pollcfg_task, &arg);
2193 	vmbus_chan_run_task(chan, &poll_cfg);
2194 }
2195 
2196 void
2197 vmbus_chan_poll_disable(struct vmbus_channel *chan)
2198 {
2199 	struct task poll_dis;
2200 
2201 	KASSERT(chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD,
2202 	    ("disable polling on non-batch chan%u", chan->ch_id));
2203 
2204 	TASK_INIT(&poll_dis, 0, vmbus_chan_polldis_task, chan);
2205 	vmbus_chan_run_task(chan, &poll_dis);
2206 }
2207