1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (c) 2009, Microsoft Corporation. 5 * 6 * Authors: 7 * Haiyang Zhang <haiyangz@microsoft.com> 8 * Hank Janssen <hjanssen@microsoft.com> 9 */ 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/wait.h> 15 #include <linux/delay.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/hyperv.h> 21 #include <linux/export.h> 22 #include <linux/io.h> 23 #include <linux/set_memory.h> 24 #include <asm/mshyperv.h> 25 26 #include "hyperv_vmbus.h" 27 28 29 struct vmbus_connection vmbus_connection = { 30 .conn_state = DISCONNECTED, 31 .unload_event = COMPLETION_INITIALIZER( 32 vmbus_connection.unload_event), 33 .next_gpadl_handle = ATOMIC_INIT(0xE1E10), 34 35 .ready_for_suspend_event = COMPLETION_INITIALIZER( 36 vmbus_connection.ready_for_suspend_event), 37 .all_offers_delivered_event = COMPLETION_INITIALIZER( 38 vmbus_connection.all_offers_delivered_event), 39 }; 40 EXPORT_SYMBOL_GPL(vmbus_connection); 41 42 /* 43 * Negotiated protocol version with the host. 44 */ 45 __u32 vmbus_proto_version; 46 EXPORT_SYMBOL_GPL(vmbus_proto_version); 47 48 /* 49 * Table of VMBus versions listed from newest to oldest. 50 * VERSION_WIN7,VERSION_WS2008, VERSION_WIN8 (which is 51 * Windows Server 2012) and VERSION_WIN8_1 (which is 52 * Windows Server 2012 R2) are no longer supported in 53 * Linux guests and are not listed. 54 */ 55 static __u32 vmbus_versions[] = { 56 VERSION_WIN10_V6_0, 57 VERSION_WIN10_V5_3, 58 VERSION_WIN10_V5_2, 59 VERSION_WIN10_V5_1, 60 VERSION_WIN10_V5, 61 VERSION_WIN10_V4_1, 62 VERSION_WIN10 63 }; 64 65 /* 66 * Maximal VMBus protocol version guests can negotiate. Useful to cap the 67 * VMBus version for testing and debugging purpose. 68 */ 69 static uint max_version = VERSION_WIN10_V6_0; 70 71 module_param(max_version, uint, S_IRUGO); 72 MODULE_PARM_DESC(max_version, 73 "Maximal VMBus protocol version which can be negotiated"); 74 75 static int vmbus_try_connection_id(struct vmbus_channel_msginfo *msginfo, 76 u32 version, u32 connection_id) 77 { 78 int ret = 0; 79 struct vmbus_channel_initiate_contact *msg; 80 unsigned long flags; 81 82 init_completion(&msginfo->waitevent); 83 84 msg = (struct vmbus_channel_initiate_contact *)msginfo->msg; 85 86 memset(msg, 0, sizeof(*msg)); 87 msg->header.msgtype = CHANNELMSG_INITIATE_CONTACT; 88 msg->vmbus_version_requested = version; 89 90 /* 91 * For VMBus protocol 5.0 (VERSION_WIN10_V5) and higher, use the 92 * caller-supplied connection_id for the Initiate Contact message so 93 * the caller can implement the required retry scheme. For subsequent 94 * messages, use the Message Connection ID field in the host-returned 95 * Version Response message. With VERSION_WIN10_V5 and higher, we don't 96 * use msg->interrupt_page, but tell the host explicitly that we still 97 * use VMBUS_MESSAGE_SINT(2) for compatibility. 98 * 99 * On old hosts, we should always use VMBUS_MESSAGE_CONNECTION_ID (1). 100 */ 101 if (version >= VERSION_WIN10_V5) { 102 msg->msg_sint = VMBUS_MESSAGE_SINT; 103 msg->msg_vtl = ms_hyperv.vtl; 104 vmbus_connection.msg_conn_id = connection_id; 105 } else { 106 msg->interrupt_page = virt_to_phys(vmbus_connection.int_page); 107 vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID; 108 } 109 110 if (vmbus_is_confidential() && version >= VERSION_WIN10_V6_0) 111 msg->feature_flags = VMBUS_FEATURE_FLAG_CONFIDENTIAL_CHANNELS; 112 113 /* 114 * shared_gpa_boundary is zero in non-SNP VMs, so it's safe to always 115 * bitwise OR it 116 */ 117 msg->monitor_page1 = virt_to_phys(vmbus_connection.monitor_pages[0]) | 118 ms_hyperv.shared_gpa_boundary; 119 msg->monitor_page2 = virt_to_phys(vmbus_connection.monitor_pages[1]) | 120 ms_hyperv.shared_gpa_boundary; 121 122 msg->target_vcpu = hv_cpu_number_to_vp_number(VMBUS_CONNECT_CPU); 123 124 /* 125 * Add to list before we send the request since we may 126 * receive the response before returning from this routine 127 */ 128 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 129 list_add_tail(&msginfo->msglistentry, 130 &vmbus_connection.chn_msg_list); 131 132 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 133 134 ret = vmbus_post_msg(msg, 135 sizeof(struct vmbus_channel_initiate_contact), 136 true); 137 138 trace_vmbus_negotiate_version(msg, ret); 139 140 if (ret != 0) { 141 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 142 list_del(&msginfo->msglistentry); 143 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, 144 flags); 145 return ret; 146 } 147 148 /* Wait for the connection response */ 149 wait_for_completion(&msginfo->waitevent); 150 151 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 152 list_del(&msginfo->msglistentry); 153 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 154 155 /* Check if successful */ 156 if (msginfo->response.version_response.version_supported) { 157 vmbus_connection.conn_state = CONNECTED; 158 159 if (version >= VERSION_WIN10_V5) 160 vmbus_connection.msg_conn_id = 161 msginfo->response.version_response.msg_conn_id; 162 } else { 163 return -ECONNREFUSED; 164 } 165 166 return ret; 167 } 168 169 int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) 170 { 171 int ret; 172 173 /* Try the redirect ID first for VTL2 with VMBus protocol 5.0+. */ 174 if (version >= VERSION_WIN10_V5 && ms_hyperv.vtl == 2) { 175 ret = vmbus_try_connection_id(msginfo, version, 176 VMBUS_MESSAGE_CONNECTION_ID_REDIRECT); 177 if (ret != -ENXIO) 178 return ret; 179 } 180 181 return vmbus_try_connection_id(msginfo, version, 182 VMBUS_MESSAGE_CONNECTION_ID_4); 183 } 184 185 /* 186 * vmbus_connect - Sends a connect request on the partition service connection 187 */ 188 int vmbus_connect(void) 189 { 190 struct vmbus_channel_msginfo *msginfo = NULL; 191 int i, ret = 0; 192 __u32 version; 193 194 /* Initialize the vmbus connection */ 195 vmbus_connection.conn_state = CONNECTING; 196 vmbus_connection.work_queue = create_workqueue("hv_vmbus_con"); 197 if (!vmbus_connection.work_queue) { 198 ret = -ENOMEM; 199 goto cleanup; 200 } 201 202 vmbus_connection.rescind_work_queue = 203 create_workqueue("hv_vmbus_rescind"); 204 if (!vmbus_connection.rescind_work_queue) { 205 ret = -ENOMEM; 206 goto cleanup; 207 } 208 vmbus_connection.ignore_any_offer_msg = false; 209 210 vmbus_connection.handle_primary_chan_wq = 211 create_workqueue("hv_pri_chan"); 212 if (!vmbus_connection.handle_primary_chan_wq) { 213 ret = -ENOMEM; 214 goto cleanup; 215 } 216 217 vmbus_connection.handle_sub_chan_wq = 218 create_workqueue("hv_sub_chan"); 219 if (!vmbus_connection.handle_sub_chan_wq) { 220 ret = -ENOMEM; 221 goto cleanup; 222 } 223 224 INIT_LIST_HEAD(&vmbus_connection.chn_msg_list); 225 spin_lock_init(&vmbus_connection.channelmsg_lock); 226 227 INIT_LIST_HEAD(&vmbus_connection.chn_list); 228 mutex_init(&vmbus_connection.channel_mutex); 229 230 /* 231 * The following Hyper-V interrupt and monitor pages can be used by 232 * UIO for mapping to user-space, so they should always be allocated on 233 * system page boundaries. The system page size must be >= the Hyper-V 234 * page size. 235 */ 236 BUILD_BUG_ON(PAGE_SIZE < HV_HYP_PAGE_SIZE); 237 238 /* 239 * Setup the vmbus event connection for channel interrupt 240 * abstraction stuff 241 */ 242 vmbus_connection.int_page = 243 (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 244 if (vmbus_connection.int_page == NULL) { 245 ret = -ENOMEM; 246 goto cleanup; 247 } 248 249 vmbus_connection.recv_int_page = vmbus_connection.int_page; 250 vmbus_connection.send_int_page = 251 (void *)((unsigned long)vmbus_connection.int_page + 252 (HV_HYP_PAGE_SIZE >> 1)); 253 254 /* 255 * Setup the monitor notification facility. The 1st page for 256 * parent->child and the 2nd page for child->parent 257 */ 258 vmbus_connection.monitor_pages[0] = (void *)__get_free_page(GFP_KERNEL); 259 vmbus_connection.monitor_pages[1] = (void *)__get_free_page(GFP_KERNEL); 260 if ((vmbus_connection.monitor_pages[0] == NULL) || 261 (vmbus_connection.monitor_pages[1] == NULL)) { 262 ret = -ENOMEM; 263 goto cleanup; 264 } 265 266 ret = set_memory_decrypted((unsigned long) 267 vmbus_connection.monitor_pages[0], 1); 268 ret |= set_memory_decrypted((unsigned long) 269 vmbus_connection.monitor_pages[1], 1); 270 if (ret) { 271 /* 272 * If set_memory_decrypted() fails, the encryption state 273 * of the memory is unknown. So leak the memory instead 274 * of risking returning decrypted memory to the free list. 275 * For simplicity, always handle both pages the same. 276 */ 277 vmbus_connection.monitor_pages[0] = NULL; 278 vmbus_connection.monitor_pages[1] = NULL; 279 goto cleanup; 280 } 281 282 /* 283 * Set_memory_decrypted() will change the memory contents if 284 * decryption occurs, so zero monitor pages here. 285 */ 286 memset(vmbus_connection.monitor_pages[0], 0x00, HV_HYP_PAGE_SIZE); 287 memset(vmbus_connection.monitor_pages[1], 0x00, HV_HYP_PAGE_SIZE); 288 289 msginfo = kzalloc(sizeof(*msginfo) + 290 sizeof(struct vmbus_channel_initiate_contact), 291 GFP_KERNEL); 292 if (msginfo == NULL) { 293 ret = -ENOMEM; 294 goto cleanup; 295 } 296 297 /* 298 * Negotiate a compatible VMBUS version number with the 299 * host. We start with the highest number we can support 300 * and work our way down until we negotiate a compatible 301 * version. 302 */ 303 304 for (i = 0; ; i++) { 305 if (i == ARRAY_SIZE(vmbus_versions)) { 306 ret = -EDOM; 307 pr_err("Hyper-V host does not support VMBus version %d.%d or higher;\n\ 308 the host may be an older version no longer supported by Linux\n", 309 vmbus_versions[i-1] >> 16, vmbus_versions[i-1] & 0xFFFF); 310 goto cleanup; 311 } 312 313 version = vmbus_versions[i]; 314 if (version > max_version) 315 continue; 316 317 ret = vmbus_negotiate_version(msginfo, version); 318 if (ret == -ETIMEDOUT) 319 goto cleanup; 320 321 if (vmbus_connection.conn_state == CONNECTED) 322 break; 323 } 324 325 if (hv_is_isolation_supported() && version < VERSION_WIN10_V5_2) { 326 pr_err("Invalid VMBus version %d.%d (expected >= %d.%d) from the host supporting isolation\n", 327 version >> 16, version & 0xFFFF, VERSION_WIN10_V5_2 >> 16, VERSION_WIN10_V5_2 & 0xFFFF); 328 ret = -EINVAL; 329 goto cleanup; 330 } 331 332 vmbus_proto_version = version; 333 pr_info("Vmbus version:%d.%d\n", 334 version >> 16, version & 0xFFFF); 335 336 vmbus_connection.channels = kzalloc_objs(struct vmbus_channel *, 337 MAX_CHANNEL_RELIDS); 338 if (vmbus_connection.channels == NULL) { 339 ret = -ENOMEM; 340 goto cleanup; 341 } 342 343 kfree(msginfo); 344 return 0; 345 346 cleanup: 347 pr_err("Unable to connect to host\n"); 348 349 vmbus_connection.conn_state = DISCONNECTED; 350 vmbus_disconnect(); 351 352 kfree(msginfo); 353 354 return ret; 355 } 356 357 void vmbus_disconnect(void) 358 { 359 /* 360 * First send the unload request to the host. 361 */ 362 vmbus_initiate_unload(false); 363 364 if (vmbus_connection.handle_sub_chan_wq) 365 destroy_workqueue(vmbus_connection.handle_sub_chan_wq); 366 367 if (vmbus_connection.handle_primary_chan_wq) 368 destroy_workqueue(vmbus_connection.handle_primary_chan_wq); 369 370 if (vmbus_connection.rescind_work_queue) 371 destroy_workqueue(vmbus_connection.rescind_work_queue); 372 373 if (vmbus_connection.work_queue) 374 destroy_workqueue(vmbus_connection.work_queue); 375 376 if (vmbus_connection.int_page) { 377 free_page((unsigned long)vmbus_connection.int_page); 378 vmbus_connection.int_page = NULL; 379 } 380 381 if (vmbus_connection.monitor_pages[0]) { 382 if (!set_memory_encrypted( 383 (unsigned long)vmbus_connection.monitor_pages[0], 1)) 384 free_page((unsigned long) 385 vmbus_connection.monitor_pages[0]); 386 vmbus_connection.monitor_pages[0] = NULL; 387 } 388 389 if (vmbus_connection.monitor_pages[1]) { 390 if (!set_memory_encrypted( 391 (unsigned long)vmbus_connection.monitor_pages[1], 1)) 392 free_page((unsigned long) 393 vmbus_connection.monitor_pages[1]); 394 vmbus_connection.monitor_pages[1] = NULL; 395 } 396 } 397 398 /* 399 * relid2channel - Get the channel object given its 400 * child relative id (ie channel id) 401 */ 402 struct vmbus_channel *relid2channel(u32 relid) 403 { 404 if (vmbus_connection.channels == NULL) { 405 pr_warn_once("relid2channel: relid=%d: No channels mapped!\n", relid); 406 return NULL; 407 } 408 if (WARN_ON(relid >= MAX_CHANNEL_RELIDS)) 409 return NULL; 410 return READ_ONCE(vmbus_connection.channels[relid]); 411 } 412 413 /* 414 * vmbus_on_event - Process a channel event notification 415 * 416 * For batched channels (default) optimize host to guest signaling 417 * by ensuring: 418 * 1. While reading the channel, we disable interrupts from host. 419 * 2. Ensure that we process all posted messages from the host 420 * before returning from this callback. 421 * 3. Once we return, enable signaling from the host. Once this 422 * state is set we check to see if additional packets are 423 * available to read. In this case we repeat the process. 424 * If this tasklet has been running for a long time 425 * then reschedule ourselves. 426 */ 427 void vmbus_on_event(unsigned long data) 428 { 429 struct vmbus_channel *channel = (void *) data; 430 void (*callback_fn)(void *context); 431 432 trace_vmbus_on_event(channel); 433 434 hv_debug_delay_test(channel, INTERRUPT_DELAY); 435 436 /* A channel once created is persistent even when 437 * there is no driver handling the device. An 438 * unloading driver sets the onchannel_callback to NULL. 439 */ 440 callback_fn = READ_ONCE(channel->onchannel_callback); 441 if (unlikely(!callback_fn)) 442 return; 443 444 (*callback_fn)(channel->channel_callback_context); 445 446 if (channel->callback_mode != HV_CALL_BATCHED) 447 return; 448 449 if (likely(hv_end_read(&channel->inbound) == 0)) 450 return; 451 452 hv_begin_read(&channel->inbound); 453 tasklet_schedule(&channel->callback_event); 454 } 455 456 /* 457 * vmbus_post_msg - Send a msg on the vmbus's message connection 458 */ 459 int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep) 460 { 461 struct vmbus_channel_message_header *hdr; 462 union hv_connection_id conn_id; 463 int ret = 0; 464 int retries = 0; 465 u32 usec = 1; 466 467 conn_id.asu32 = 0; 468 conn_id.u.id = vmbus_connection.msg_conn_id; 469 470 /* 471 * hv_post_message() can have transient failures because of 472 * insufficient resources. Retry the operation a couple of 473 * times before giving up. 474 */ 475 while (retries < 100) { 476 ret = hv_post_message(conn_id, 1, buffer, buflen); 477 478 switch (ret) { 479 case HV_STATUS_INVALID_CONNECTION_ID: 480 /* Allow INITIATE_CONTACT to try another connection ID. */ 481 hdr = buffer; 482 if (hdr->msgtype == CHANNELMSG_INITIATE_CONTACT) 483 return -ENXIO; 484 /* 485 * We could get this if we send messages too 486 * frequently. 487 */ 488 ret = -EAGAIN; 489 break; 490 case HV_STATUS_INSUFFICIENT_MEMORY: 491 case HV_STATUS_INSUFFICIENT_BUFFERS: 492 ret = -ENOBUFS; 493 break; 494 case HV_STATUS_SUCCESS: 495 return ret; 496 default: 497 pr_err("hv_post_msg() failed; error code:%d\n", ret); 498 return -EINVAL; 499 } 500 501 retries++; 502 if (can_sleep && usec > 1000) 503 msleep(usec / 1000); 504 else if (usec < MAX_UDELAY_MS * 1000) 505 udelay(usec); 506 else 507 mdelay(usec / 1000); 508 509 if (retries < 22) 510 usec *= 2; 511 } 512 return ret; 513 } 514 515 /* 516 * vmbus_set_event - Send an event notification to the parent 517 */ 518 void vmbus_set_event(struct vmbus_channel *channel) 519 { 520 u32 child_relid = channel->offermsg.child_relid; 521 522 if (!channel->is_dedicated_interrupt) 523 vmbus_send_interrupt(child_relid); 524 525 ++channel->sig_events; 526 527 if (ms_hyperv.paravisor_present) { 528 if (hv_isolation_type_snp()) 529 hv_ghcb_hypercall(HVCALL_SIGNAL_EVENT, &channel->sig_event, 530 NULL, sizeof(channel->sig_event)); 531 else if (hv_isolation_type_tdx()) 532 hv_tdx_hypercall(HVCALL_SIGNAL_EVENT | HV_HYPERCALL_FAST_BIT, 533 channel->sig_event, 0); 534 else 535 WARN_ON_ONCE(1); 536 } else { 537 u64 control = HVCALL_SIGNAL_EVENT; 538 539 control |= hv_nested ? HV_HYPERCALL_NESTED : 0; 540 hv_do_fast_hypercall8(control, channel->sig_event); 541 } 542 } 543 EXPORT_SYMBOL_GPL(vmbus_set_event); 544