1 // SPDX-License-Identifier: GPL-2.0-only
148 	 * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart
180 	channel->rescind = true;  in vmbus_rescind_cleanup()
184 		if (msginfo->waiting_channel == channel) {  in vmbus_rescind_cleanup()
185 			complete(&msginfo->waitevent);  in vmbus_rescind_cleanup()
204 	const guid_t *guid = &channel->offermsg.offer.if_type;  in hv_get_dev_type()
219  * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
232  * Set up and fill in default negotiate response message.
233  * Mainly used by Hyper-V drivers.
254 	icmsghdrp->icmsgsize = 0x10;  in vmbus_prep_negotiate_resp()
257 	icframe_major = negop->icframe_vercnt;  in vmbus_prep_negotiate_resp()
260 	icmsg_major = negop->icmsg_vercnt;  in vmbus_prep_negotiate_resp()
267 		pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",  in vmbus_prep_negotiate_resp()
281 		for (j = 0; j < negop->icframe_vercnt; j++) {  in vmbus_prep_negotiate_resp()
282 			if ((negop->icversion_data[j].major == fw_major) &&  in vmbus_prep_negotiate_resp()
283 			    (negop->icversion_data[j].minor == fw_minor)) {  in vmbus_prep_negotiate_resp()
284 				icframe_major = negop->icversion_data[j].major;  in vmbus_prep_negotiate_resp()
285 				icframe_minor = negop->icversion_data[j].minor;  in vmbus_prep_negotiate_resp()
304 		for (j = negop->icframe_vercnt;  in vmbus_prep_negotiate_resp()
305 			(j < negop->icframe_vercnt + negop->icmsg_vercnt);  in vmbus_prep_negotiate_resp()
308 			if ((negop->icversion_data[j].major == srv_major) &&  in vmbus_prep_negotiate_resp()
309 				(negop->icversion_data[j].minor == srv_minor)) {  in vmbus_prep_negotiate_resp()
311 				icmsg_major = negop->icversion_data[j].major;  in vmbus_prep_negotiate_resp()
312 				icmsg_minor = negop->icversion_data[j].minor;  in vmbus_prep_negotiate_resp()
329 		negop->icframe_vercnt = 0;  in vmbus_prep_negotiate_resp()
330 		negop->icmsg_vercnt = 0;  in vmbus_prep_negotiate_resp()
332 		negop->icframe_vercnt = 1;  in vmbus_prep_negotiate_resp()
333 		negop->icmsg_vercnt = 1;  in vmbus_prep_negotiate_resp()
342 	negop->icversion_data[0].major = icframe_major;  in vmbus_prep_negotiate_resp()
343 	negop->icversion_data[0].minor = icframe_minor;  in vmbus_prep_negotiate_resp()
344 	negop->icversion_data[1].major = icmsg_major;  in vmbus_prep_negotiate_resp()
345 	negop->icversion_data[1].minor = icmsg_minor;  in vmbus_prep_negotiate_resp()
351  * alloc_channel - Allocate and initialize a vmbus channel object
361 	spin_lock_init(&channel->sched_lock);  in alloc_channel()
362 	init_completion(&channel->rescind_event);  in alloc_channel()
364 	INIT_LIST_HEAD(&channel->sc_list);  in alloc_channel()
366 	tasklet_init(&channel->callback_event,  in alloc_channel()
375  * free_channel - Release the resources used by the vmbus channel object
379 	tasklet_kill(&channel->callback_event);  in free_channel()
382 	kobject_put(&channel->kobj);  in free_channel()
387 	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))  in vmbus_channel_map_relid()
399 	 *      OPENCHANNEL message for the channel is sent in vmbus_open().  in vmbus_channel_map_relid()
400 	 *      Hyper-V won't start sending the interrupts for the channel  in vmbus_channel_map_relid()
401 	 *      before the OPENCHANNEL message is acked.  The memory barrier  in vmbus_channel_map_relid()
402 	 *      in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures  in vmbus_channel_map_relid()
415 		vmbus_connection.channels[channel->offermsg.child_relid],  in vmbus_channel_map_relid()
421 	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))  in vmbus_channel_unmap_relid()
424 		vmbus_connection.channels[channel->offermsg.child_relid],  in vmbus_channel_unmap_relid()
445 	BUG_ON(!channel->rescind);  in hv_process_channel_removal()
451 	WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&  in hv_process_channel_removal()
455 	 * Upon suspend, an in-use hv_sock channel is removed from the array of  in hv_process_channel_removal()
457 	 * user-space application destroys the channel, it's unnecessary and  in hv_process_channel_removal()
461 	if (channel->offermsg.child_relid != INVALID_RELID)  in hv_process_channel_removal()
464 	if (channel->primary_channel == NULL)  in hv_process_channel_removal()
465 		list_del(&channel->listentry);  in hv_process_channel_removal()
467 		list_del(&channel->sc_list);  in hv_process_channel_removal()
471 	 * init_vp_index() can (re-)use the CPU.  in hv_process_channel_removal()
474 		hv_clear_allocated_cpu(channel->target_cpu);  in hv_process_channel_removal()
477 	 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and  in hv_process_channel_removal()
478 	 * the relid is invalidated; after hibernation, when the user-space app  in hv_process_channel_removal()
483 	if (channel->offermsg.child_relid != INVALID_RELID)  in hv_process_channel_removal()
484 		vmbus_release_relid(channel->offermsg.child_relid);  in hv_process_channel_removal()
496 		channel->rescind = true;  in vmbus_free_channels()
498 		vmbus_device_unregister(channel->device_obj);  in vmbus_free_channels()
502 /* Note: the function can run concurrently for primary/sub channels. */
507 	struct vmbus_channel *primary_channel = newchannel->primary_channel;  in vmbus_add_channel_work()
515 	newchannel->state = CHANNEL_OPEN_STATE;  in vmbus_add_channel_work()
518 		/* newchannel is a sub-channel. */  in vmbus_add_channel_work()
519 		struct hv_device *dev = primary_channel->device_obj;  in vmbus_add_channel_work()
524 		if (primary_channel->sc_creation_callback != NULL)  in vmbus_add_channel_work()
525 			primary_channel->sc_creation_callback(newchannel);  in vmbus_add_channel_work()
527 		newchannel->probe_done = true;  in vmbus_add_channel_work()
534 	newchannel->device_obj = vmbus_device_create(  in vmbus_add_channel_work()
535 		&newchannel->offermsg.offer.if_type,  in vmbus_add_channel_work()
536 		&newchannel->offermsg.offer.if_instance,  in vmbus_add_channel_work()
538 	if (!newchannel->device_obj)  in vmbus_add_channel_work()
541 	newchannel->device_obj->device_id = newchannel->device_id;  in vmbus_add_channel_work()
543 	 * Add the new device to the bus. This will kick off device-driver  in vmbus_add_channel_work()
552 	ret = vmbus_device_register(newchannel->device_obj);  in vmbus_add_channel_work()
556 			newchannel->offermsg.child_relid);  in vmbus_add_channel_work()
560 	newchannel->probe_done = true;  in vmbus_add_channel_work()
570 	newchannel->probe_done = true;  in vmbus_add_channel_work()
573 		list_del(&newchannel->listentry);  in vmbus_add_channel_work()
575 		list_del(&newchannel->sc_list);  in vmbus_add_channel_work()
582 	vmbus_release_relid(newchannel->offermsg.child_relid);  in vmbus_add_channel_work()
588  * vmbus_process_offer - Process the offer by creating a channel/device
625 		if (guid_equal(&channel->offermsg.offer.if_type,  in vmbus_process_offer()
626 			       &newchannel->offermsg.offer.if_type) &&  in vmbus_process_offer()
627 		    guid_equal(&channel->offermsg.offer.if_instance,  in vmbus_process_offer()
628 			       &newchannel->offermsg.offer.if_instance)) {  in vmbus_process_offer()
630 			newchannel->primary_channel = channel;  in vmbus_process_offer()
648 		list_add_tail(&newchannel->listentry,  in vmbus_process_offer()
652 		 * Check to see if this is a valid sub-channel.  in vmbus_process_offer()
654 		if (newchannel->offermsg.offer.sub_channel_index == 0) {  in vmbus_process_offer()
658 			 * Don't call free_channel(), because newchannel->kobj  in vmbus_process_offer()
666 		 * Process the sub-channel.  in vmbus_process_offer()
668 		list_add_tail(&newchannel->sc_list, &channel->sc_list);  in vmbus_process_offer()
677 	 * vmbus_process_offer() mustn't call channel->sc_creation_callback()  in vmbus_process_offer()
678 	 * directly for sub-channels, because sc_creation_callback() ->  in vmbus_process_offer()
680 	 * OPEN_CHANNEL message (the host may rescind a channel at any time,  in vmbus_process_offer()
682 	 * may not wake up the vmbus_open() as it's blocked due to a non-zero  in vmbus_process_offer()
688 	 * And, usually the handling of primary channels and sub-channels can  in vmbus_process_offer()
690 	 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,  in vmbus_process_offer()
691 	 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->  in vmbus_process_offer()
693 	 * and waits for all the sub-channels to appear, but the latter  in vmbus_process_offer()
695 	 * sub-channels.  in vmbus_process_offer()
697 	INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);  in vmbus_process_offer()
700 	queue_work(wq, &newchannel->add_channel_work);  in vmbus_process_offer()
709 	struct vmbus_channel *primary = chn->primary_channel;  in hv_cpuself_used()
717 	if (primary->target_cpu == cpu)  in hv_cpuself_used()
720 	list_for_each_entry(sc, &primary->sc_list, sc_list)  in hv_cpuself_used()
721 		if (sc != chn && sc->target_cpu == cpu)  in hv_cpuself_used()
736  * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
762 		channel->target_cpu = VMBUS_CONNECT_CPU;  in init_vp_index()
797 		if (channel->offermsg.offer.sub_channel_index >= ncpu ||  in init_vp_index()
802 	channel->target_cpu = target_cpu;  in init_vp_index()
828 	 * read message pages for all CPUs directly.  in vmbus_wait_for_unload()
836 	 * message isn't seen.  in vmbus_wait_for_unload()
851 			 * not all present CPUs are online, the message page  in vmbus_wait_for_unload()
854 			page_addr = hv_cpu->synic_message_page;  in vmbus_wait_for_unload()
861 			message_type = READ_ONCE(msg->header.message_type);  in vmbus_wait_for_unload()
866 				msg->u.payload;  in vmbus_wait_for_unload()
868 			if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)  in vmbus_wait_for_unload()
888 	 * maybe-pending messages on all CPUs to be able to receive new  in vmbus_wait_for_unload()
895 		page_addr = hv_cpu->synic_message_page;  in vmbus_wait_for_unload()
900 		msg->header.message_type = HVMSG_NONE;  in vmbus_wait_for_unload()
905  * vmbus_unload_response - Handler for the unload response.
913 	 * NB.  A malicious or compromised Hyper-V could send a spurious  in vmbus_unload_response()
914 	 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call  in vmbus_unload_response()
928 	/* Pre-Win2012R2 hosts don't support reconnect */  in vmbus_initiate_unload()
954 	channel->sig_event = VMBUS_EVENT_CONNECTION_ID;  in vmbus_setup_channel_state()
956 	channel->is_dedicated_interrupt =  in vmbus_setup_channel_state()
957 			(offer->is_dedicated_interrupt != 0);  in vmbus_setup_channel_state()
958 	channel->sig_event = offer->connection_id;  in vmbus_setup_channel_state()
960 	memcpy(&channel->offermsg, offer,  in vmbus_setup_channel_state()
962 	channel->monitor_grp = (u8)offer->monitorid / 32;  in vmbus_setup_channel_state()
963 	channel->monitor_bit = (u8)offer->monitorid % 32;  in vmbus_setup_channel_state()
964 	channel->device_id = hv_get_dev_type(channel);  in vmbus_setup_channel_state()
968  * find_primary_channel_by_offer - Get the channel object given the new offer.
977 	/* Ignore sub-channel offers. */  in find_primary_channel_by_offer()
978 	if (offer->offer.sub_channel_index != 0)  in find_primary_channel_by_offer()
984 		inst1 = &iter->offermsg.offer.if_instance;  in find_primary_channel_by_offer()
985 		inst2 = &offer->offer.if_instance;  in find_primary_channel_by_offer()
1000 	const guid_t *guid = &offer->offer.if_type;  in vmbus_is_valid_offer()
1017  * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1032 				   offer->child_relid);  in vmbus_onoffer()
1041 		 * We're resuming from hibernation: all the sub-channel and  in vmbus_onoffer()
1043 		 * been cleaned up, and now we must be seeing a re-offered  in vmbus_onoffer()
1066 		 * suspend are re-offered upon the resume.  See the WARN_ON()  in vmbus_onoffer()
1073 		WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);  in vmbus_onoffer()
1075 		oldchannel->offermsg.child_relid = offer->child_relid;  in vmbus_onoffer()
1078 		if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {  in vmbus_onoffer()
1082 			 * (Build 17763), the offer->connection_id of the  in vmbus_onoffer()
1087 				 offer->child_relid);  in vmbus_onoffer()
1091 					     &oldchannel->offermsg, offer_sz,  in vmbus_onoffer()
1110 		vmbus_release_relid(offer->child_relid);  in vmbus_onoffer()
1124 	 * If all the sub-channels or hv_sock channels have been cleaned up,  in check_ready_for_suspend_event()
1132  * vmbus_onoffer_rescind - Rescind offer handler.
1149 	 * from the host are guranteed to be ordered -  in vmbus_onoffer_rescind()
1180 	channel = relid2channel(rescind->child_relid);  in vmbus_onoffer_rescind()
1187 		if (channel->rescind_ref) {  in vmbus_onoffer_rescind()
1191 		channel->rescind_ref = true;  in vmbus_onoffer_rescind()
1197 		 * We failed in processing the offer message;  in vmbus_onoffer_rescind()
1207 	 * Before setting channel->rescind in vmbus_rescind_cleanup(), we  in vmbus_onoffer_rescind()
1216 	while (READ_ONCE(channel->probe_done) == false) {  in vmbus_onoffer_rescind()
1228 	if (channel->device_obj) {  in vmbus_onoffer_rescind()
1229 		if (channel->chn_rescind_callback) {  in vmbus_onoffer_rescind()
1230 			channel->chn_rescind_callback(channel);  in vmbus_onoffer_rescind()
1241 		dev = get_device(&channel->device_obj->device);  in vmbus_onoffer_rescind()
1243 			vmbus_device_unregister(channel->device_obj);  in vmbus_onoffer_rescind()
1246 	} else if (channel->primary_channel != NULL) {  in vmbus_onoffer_rescind()
1248 		 * Sub-channel is being rescinded. Following is the channel  in vmbus_onoffer_rescind()
1251 		 * 1. Close all sub-channels first  in vmbus_onoffer_rescind()
1255 		if (channel->state == CHANNEL_OPEN_STATE) {  in vmbus_onoffer_rescind()
1262 			complete(&channel->rescind_event);  in vmbus_onoffer_rescind()
1278 	while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))  in vmbus_hvsock_device_unregister()
1281 	vmbus_device_unregister(channel->device_obj);  in vmbus_hvsock_device_unregister()
1287  * vmbus_onoffers_delivered -
1288  * The CHANNELMSG_ALLOFFERS_DELIVERED message arrives after all
1289  * boot-time offers are delivered. A boot-time offer is for the primary
1291  * Boot-time offers include offers for physical devices assigned to the VM
1292  * via Hyper-V's Discrete Device Assignment (DDA) functionality that are
1294  * Boot-time offers do not include offers for VMBus sub-channels. Because
1295  * devices can be hot-added to the VM after it is booted, additional channel
1296  * offers that aren't boot-time offers can be received at any time after the
1297  * all-offers-delivered message.
1299  * SR-IOV NIC Virtual Functions (VFs) assigned to a VM are not considered
1300  * to be assigned to the VM at boot-time, and offers for VFs may occur after
1301  * the all-offers-delivered message. VFs are optional accelerators to the
1302  * synthetic VMBus NIC and are effectively hot-added only after the VMBus
1313  * vmbus_onopen_result - Open result handler.
1339 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_onopen_result()
1341 		if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {  in vmbus_onopen_result()
1343 			(struct vmbus_channel_open_channel *)msginfo->msg;  in vmbus_onopen_result()
1344 			if (openmsg->child_relid == result->child_relid &&  in vmbus_onopen_result()
1345 			    openmsg->openid == result->openid) {  in vmbus_onopen_result()
1346 				memcpy(&msginfo->response.open_result,  in vmbus_onopen_result()
1350 				complete(&msginfo->waitevent);  in vmbus_onopen_result()
1359  * vmbus_ongpadl_created - GPADL created handler.
1386 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_ongpadl_created()
1388 		if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {  in vmbus_ongpadl_created()
1392 			if ((gpadlcreated->child_relid ==  in vmbus_ongpadl_created()
1393 			     gpadlheader->child_relid) &&  in vmbus_ongpadl_created()
1394 			    (gpadlcreated->gpadl == gpadlheader->gpadl)) {  in vmbus_ongpadl_created()
1395 				memcpy(&msginfo->response.gpadl_created,  in vmbus_ongpadl_created()
1399 				complete(&msginfo->waitevent);  in vmbus_ongpadl_created()
1408  * vmbus_onmodifychannel_response - Modify Channel response handler.
1430 				(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_onmodifychannel_response()
1432 		if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {  in vmbus_onmodifychannel_response()
1435 			modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;  in vmbus_onmodifychannel_response()
1436 			if (modifymsg->child_relid == response->child_relid) {  in vmbus_onmodifychannel_response()
1437 				memcpy(&msginfo->response.modify_response, response,  in vmbus_onmodifychannel_response()
1439 				complete(&msginfo->waitevent);  in vmbus_onmodifychannel_response()
1448  * vmbus_ongpadl_torndown - GPADL torndown handler.
1475 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_ongpadl_torndown()
1477 		if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {  in vmbus_ongpadl_torndown()
1481 			if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {  in vmbus_ongpadl_torndown()
1482 				memcpy(&msginfo->response.gpadl_torndown,  in vmbus_ongpadl_torndown()
1486 				complete(&msginfo->waitevent);  in vmbus_ongpadl_torndown()
1495  * vmbus_onversion_response - Version response handler
1518 			(struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_onversion_response()
1520 		if (requestheader->msgtype ==  in vmbus_onversion_response()
1522 			memcpy(&msginfo->response.version_response,  in vmbus_onversion_response()
1525 			complete(&msginfo->waitevent);  in vmbus_onversion_response()
1531 /* Channel message dispatch table */
1569  * vmbus_onmessage - Handler for channel protocol messages.
1578 	 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go  in vmbus_onmessage()
1581 	channel_message_table[hdr->msgtype].message_handler(hdr);  in vmbus_onmessage()
1585  * vmbus_request_offers - Send a request to get all our pending offers
1586  * and wait for all boot-time offers to arrive.
1598 		return -ENOMEM;  in vmbus_request_offers()
1600 	msg = (struct vmbus_channel_message_header *)msginfo->msg;  in vmbus_request_offers()
1602 	msg->msgtype = CHANNELMSG_REQUESTOFFERS;  in vmbus_request_offers()
1605 	 * This REQUESTOFFERS message will result in the host sending an all  in vmbus_request_offers()
1606 	 * offers delivered message after all the boot-time offers are sent.  in vmbus_request_offers()
1614 		pr_err("Unable to request offers - %d\n", ret);  in vmbus_request_offers()
1620 	 * Wait for the host to send all boot-time offers.  in vmbus_request_offers()
1621 	 * Keeping it as a best-effort mechanism, where a warning is  in vmbus_request_offers()
1626 		pr_warn("timed out waiting for all boot-time offers to be delivered.\n");  in vmbus_request_offers()
1651 	primary_channel->sc_creation_callback = sc_cr_cb;  in vmbus_set_sc_create_callback()
1658 	channel->chn_rescind_callback = chn_rescind_cb;  in vmbus_set_chn_rescind_callback()