1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2016 VMware, Inc., Palo Alto, CA., USA 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial portions 15 * of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 20 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 23 * USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <linux/objtool.h> 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/slab.h> 31 #include <linux/cc_platform.h> 32 33 #include <asm/hypervisor.h> 34 #include <drm/drm_ioctl.h> 35 36 #include "vmwgfx_drv.h" 37 #include "vmwgfx_msg_x86.h" 38 #include "vmwgfx_msg_arm64.h" 39 #include "vmwgfx_mksstat.h" 40 41 #define MESSAGE_STATUS_SUCCESS 0x0001 42 #define MESSAGE_STATUS_DORECV 0x0002 43 #define MESSAGE_STATUS_CPT 0x0010 44 #define MESSAGE_STATUS_HB 0x0080 45 46 #define RPCI_PROTOCOL_NUM 0x49435052 47 #define GUESTMSG_FLAG_COOKIE 0x80000000 48 49 #define RETRIES 3 50 51 #define VMW_PORT_CMD_MSG 30 52 #define VMW_PORT_CMD_HB_MSG 0 53 #define VMW_PORT_CMD_OPEN_CHANNEL (MSG_TYPE_OPEN << 16 | VMW_PORT_CMD_MSG) 54 #define VMW_PORT_CMD_CLOSE_CHANNEL (MSG_TYPE_CLOSE << 16 | VMW_PORT_CMD_MSG) 55 #define VMW_PORT_CMD_SENDSIZE (MSG_TYPE_SENDSIZE << 16 | VMW_PORT_CMD_MSG) 56 #define VMW_PORT_CMD_RECVSIZE (MSG_TYPE_RECVSIZE << 16 | VMW_PORT_CMD_MSG) 57 #define VMW_PORT_CMD_RECVSTATUS (MSG_TYPE_RECVSTATUS << 16 | VMW_PORT_CMD_MSG) 58 59 #define VMW_PORT_CMD_MKS_GUEST_STATS 85 60 #define VMW_PORT_CMD_MKSGS_RESET (0 << 16 | VMW_PORT_CMD_MKS_GUEST_STATS) 61 #define VMW_PORT_CMD_MKSGS_ADD_PPN (1 << 16 | VMW_PORT_CMD_MKS_GUEST_STATS) 62 #define VMW_PORT_CMD_MKSGS_REMOVE_PPN (2 << 16 | VMW_PORT_CMD_MKS_GUEST_STATS) 63 64 #define HIGH_WORD(X) ((X & 0xFFFF0000) >> 16) 65 66 #define MAX_USER_MSG_LENGTH PAGE_SIZE 67 68 static u32 vmw_msg_enabled = 1; 69 70 enum rpc_msg_type { 71 MSG_TYPE_OPEN, 72 MSG_TYPE_SENDSIZE, 73 MSG_TYPE_SENDPAYLOAD, 74 MSG_TYPE_RECVSIZE, 75 MSG_TYPE_RECVPAYLOAD, 76 MSG_TYPE_RECVSTATUS, 77 MSG_TYPE_CLOSE, 78 }; 79 80 struct rpc_channel { 81 u16 channel_id; 82 u32 cookie_high; 83 u32 cookie_low; 84 }; 85 86 #if IS_ENABLED(CONFIG_DRM_VMWGFX_MKSSTATS) 87 /* Kernel mksGuestStats counter names and desciptions; same order as enum mksstat_kern_stats_t */ 88 static const char* const mksstat_kern_name_desc[MKSSTAT_KERN_COUNT][2] = 89 { 90 { "vmw_execbuf_ioctl", "vmw_execbuf_ioctl" }, 91 { "vmw_cotable_resize", "vmw_cotable_resize" }, 92 }; 93 #endif 94 95 /** 96 * vmw_open_channel 97 * 98 * @channel: RPC channel 99 * @protocol: 100 * 101 * Returns: 0 on success 102 */ 103 static int vmw_open_channel(struct rpc_channel *channel, unsigned int protocol) 104 { 105 u32 ecx, edx, esi, edi; 106 107 vmware_hypercall6(VMW_PORT_CMD_OPEN_CHANNEL, 108 (protocol | GUESTMSG_FLAG_COOKIE), 0, 109 &ecx, &edx, &esi, &edi); 110 111 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) 112 return -EINVAL; 113 114 channel->channel_id = HIGH_WORD(edx); 115 channel->cookie_high = esi; 116 channel->cookie_low = edi; 117 118 return 0; 119 } 120 121 122 123 /** 124 * vmw_close_channel 125 * 126 * @channel: RPC channel 127 * 128 * Returns: 0 on success 129 */ 130 static int vmw_close_channel(struct rpc_channel *channel) 131 { 132 u32 ecx; 133 134 vmware_hypercall5(VMW_PORT_CMD_CLOSE_CHANNEL, 135 0, channel->channel_id << 16, 136 channel->cookie_high, 137 channel->cookie_low, 138 &ecx); 139 140 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) 141 return -EINVAL; 142 143 return 0; 144 } 145 146 /** 147 * vmw_port_hb_out - Send the message payload either through the 148 * high-bandwidth port if available, or through the backdoor otherwise. 149 * @channel: The rpc channel. 150 * @msg: NULL-terminated message. 151 * @hb: Whether the high-bandwidth port is available. 152 * 153 * Return: The port status. 154 */ 155 static unsigned long vmw_port_hb_out(struct rpc_channel *channel, 156 const char *msg, bool hb) 157 { 158 u32 ebx, ecx; 159 unsigned long msg_len = strlen(msg); 160 161 /* HB port can't access encrypted memory. */ 162 if (hb && !cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { 163 vmware_hypercall_hb_out( 164 (MESSAGE_STATUS_SUCCESS << 16) | VMW_PORT_CMD_HB_MSG, 165 msg_len, 166 channel->channel_id << 16, 167 (uintptr_t) msg, channel->cookie_low, 168 channel->cookie_high, 169 &ebx); 170 171 return ebx; 172 } 173 174 /* HB port not available. Send the message 4 bytes at a time. */ 175 ecx = MESSAGE_STATUS_SUCCESS << 16; 176 while (msg_len && (HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS)) { 177 unsigned int bytes = min_t(size_t, msg_len, 4); 178 unsigned long word = 0; 179 180 memcpy(&word, msg, bytes); 181 msg_len -= bytes; 182 msg += bytes; 183 184 vmware_hypercall5(VMW_PORT_CMD_MSG | 185 (MSG_TYPE_SENDPAYLOAD << 16), 186 word, channel->channel_id << 16, 187 channel->cookie_high, 188 channel->cookie_low, 189 &ecx); 190 } 191 192 return ecx; 193 } 194 195 /** 196 * vmw_port_hb_in - Receive the message payload either through the 197 * high-bandwidth port if available, or through the backdoor otherwise. 198 * @channel: The rpc channel. 199 * @reply: Pointer to buffer holding reply. 200 * @reply_len: Length of the reply. 201 * @hb: Whether the high-bandwidth port is available. 202 * 203 * Return: The port status. 204 */ 205 static unsigned long vmw_port_hb_in(struct rpc_channel *channel, char *reply, 206 unsigned long reply_len, bool hb) 207 { 208 u32 ebx, ecx, edx; 209 210 /* HB port can't access encrypted memory */ 211 if (hb && !cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { 212 vmware_hypercall_hb_in( 213 (MESSAGE_STATUS_SUCCESS << 16) | VMW_PORT_CMD_HB_MSG, 214 reply_len, 215 channel->channel_id << 16, 216 channel->cookie_high, 217 (uintptr_t) reply, channel->cookie_low, 218 &ebx); 219 220 return ebx; 221 } 222 223 /* HB port not available. Retrieve the message 4 bytes at a time. */ 224 ecx = MESSAGE_STATUS_SUCCESS << 16; 225 while (reply_len) { 226 unsigned int bytes = min_t(unsigned long, reply_len, 4); 227 228 vmware_hypercall7(VMW_PORT_CMD_MSG | 229 (MSG_TYPE_RECVPAYLOAD << 16), 230 MESSAGE_STATUS_SUCCESS, 231 channel->channel_id << 16, 232 channel->cookie_high, 233 channel->cookie_low, 234 &ebx, &ecx, &edx); 235 236 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) 237 break; 238 239 memcpy(reply, &ebx, bytes); 240 reply_len -= bytes; 241 reply += bytes; 242 } 243 244 return ecx; 245 } 246 247 248 /** 249 * vmw_send_msg: Sends a message to the host 250 * 251 * @channel: RPC channel 252 * @msg: NULL terminated string 253 * 254 * Returns: 0 on success 255 */ 256 static int vmw_send_msg(struct rpc_channel *channel, const char *msg) 257 { 258 u32 ebx, ecx; 259 size_t msg_len = strlen(msg); 260 int retries = 0; 261 262 while (retries < RETRIES) { 263 retries++; 264 265 vmware_hypercall5(VMW_PORT_CMD_SENDSIZE, 266 msg_len, channel->channel_id << 16, 267 channel->cookie_high, 268 channel->cookie_low, 269 &ecx); 270 271 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) { 272 /* Expected success. Give up. */ 273 return -EINVAL; 274 } 275 276 /* Send msg */ 277 ebx = vmw_port_hb_out(channel, msg, 278 !!(HIGH_WORD(ecx) & MESSAGE_STATUS_HB)); 279 280 if ((HIGH_WORD(ebx) & MESSAGE_STATUS_SUCCESS) != 0) { 281 return 0; 282 } else if ((HIGH_WORD(ebx) & MESSAGE_STATUS_CPT) != 0) { 283 /* A checkpoint occurred. Retry. */ 284 continue; 285 } else { 286 break; 287 } 288 } 289 290 return -EINVAL; 291 } 292 STACK_FRAME_NON_STANDARD(vmw_send_msg); 293 294 295 /** 296 * vmw_recv_msg: Receives a message from the host 297 * 298 * Note: It is the caller's responsibility to call kfree() on msg. 299 * 300 * @channel: channel opened by vmw_open_channel 301 * @msg: [OUT] message received from the host 302 * @msg_len: message length 303 */ 304 static int vmw_recv_msg(struct rpc_channel *channel, void **msg, 305 size_t *msg_len) 306 { 307 u32 ebx, ecx, edx; 308 char *reply; 309 size_t reply_len; 310 int retries = 0; 311 312 313 *msg_len = 0; 314 *msg = NULL; 315 316 while (retries < RETRIES) { 317 retries++; 318 319 vmware_hypercall7(VMW_PORT_CMD_RECVSIZE, 320 0, channel->channel_id << 16, 321 channel->cookie_high, 322 channel->cookie_low, 323 &ebx, &ecx, &edx); 324 325 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) { 326 DRM_ERROR("Failed to get reply size for host message.\n"); 327 return -EINVAL; 328 } 329 330 /* No reply available. This is okay. */ 331 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_DORECV) == 0) 332 return 0; 333 334 reply_len = ebx; 335 reply = kzalloc(reply_len + 1, GFP_KERNEL); 336 if (!reply) { 337 DRM_ERROR("Cannot allocate memory for host message reply.\n"); 338 return -ENOMEM; 339 } 340 341 342 /* Receive buffer */ 343 ebx = vmw_port_hb_in(channel, reply, reply_len, 344 !!(HIGH_WORD(ecx) & MESSAGE_STATUS_HB)); 345 if ((HIGH_WORD(ebx) & MESSAGE_STATUS_SUCCESS) == 0) { 346 kfree(reply); 347 reply = NULL; 348 if ((HIGH_WORD(ebx) & MESSAGE_STATUS_CPT) != 0) { 349 /* A checkpoint occurred. Retry. */ 350 continue; 351 } 352 353 return -EINVAL; 354 } 355 356 reply[reply_len] = '\0'; 357 358 vmware_hypercall5(VMW_PORT_CMD_RECVSTATUS, 359 MESSAGE_STATUS_SUCCESS, 360 channel->channel_id << 16, 361 channel->cookie_high, 362 channel->cookie_low, 363 &ecx); 364 365 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_SUCCESS) == 0) { 366 kfree(reply); 367 reply = NULL; 368 if ((HIGH_WORD(ecx) & MESSAGE_STATUS_CPT) != 0) { 369 /* A checkpoint occurred. Retry. */ 370 continue; 371 } 372 373 return -EINVAL; 374 } 375 376 break; 377 } 378 379 if (!reply) 380 return -EINVAL; 381 382 *msg_len = reply_len; 383 *msg = reply; 384 385 return 0; 386 } 387 STACK_FRAME_NON_STANDARD(vmw_recv_msg); 388 389 390 /** 391 * vmw_host_get_guestinfo: Gets a GuestInfo parameter 392 * 393 * Gets the value of a GuestInfo.* parameter. The value returned will be in 394 * a string, and it is up to the caller to post-process. 395 * 396 * @guest_info_param: Parameter to get, e.g. GuestInfo.svga.gl3 397 * @buffer: if NULL, *reply_len will contain reply size. 398 * @length: size of the reply_buf. Set to size of reply upon return 399 * 400 * Returns: 0 on success 401 */ 402 int vmw_host_get_guestinfo(const char *guest_info_param, 403 char *buffer, size_t *length) 404 { 405 struct rpc_channel channel; 406 char *msg, *reply = NULL; 407 size_t reply_len = 0; 408 409 if (!vmw_msg_enabled) 410 return -ENODEV; 411 412 if (!guest_info_param || !length) 413 return -EINVAL; 414 415 msg = kasprintf(GFP_KERNEL, "info-get %s", guest_info_param); 416 if (!msg) { 417 DRM_ERROR("Cannot allocate memory to get guest info \"%s\".", 418 guest_info_param); 419 return -ENOMEM; 420 } 421 422 if (vmw_open_channel(&channel, RPCI_PROTOCOL_NUM)) 423 goto out_open; 424 425 if (vmw_send_msg(&channel, msg) || 426 vmw_recv_msg(&channel, (void *) &reply, &reply_len)) 427 goto out_msg; 428 429 vmw_close_channel(&channel); 430 if (buffer && reply && reply_len > 0) { 431 /* Remove reply code, which are the first 2 characters of 432 * the reply 433 */ 434 reply_len = max(reply_len - 2, (size_t) 0); 435 reply_len = min(reply_len, *length); 436 437 if (reply_len > 0) 438 memcpy(buffer, reply + 2, reply_len); 439 } 440 441 *length = reply_len; 442 443 kfree(reply); 444 kfree(msg); 445 446 return 0; 447 448 out_msg: 449 vmw_close_channel(&channel); 450 kfree(reply); 451 out_open: 452 *length = 0; 453 kfree(msg); 454 DRM_ERROR("Failed to get guest info \"%s\".", guest_info_param); 455 456 return -EINVAL; 457 } 458 459 460 /** 461 * vmw_host_printf: Sends a log message to the host 462 * 463 * @fmt: Regular printf format string and arguments 464 * 465 * Returns: 0 on success 466 */ 467 __printf(1, 2) 468 int vmw_host_printf(const char *fmt, ...) 469 { 470 va_list ap; 471 struct rpc_channel channel; 472 char *msg; 473 char *log; 474 int ret = 0; 475 476 if (!vmw_msg_enabled) 477 return -ENODEV; 478 479 if (!fmt) 480 return ret; 481 482 va_start(ap, fmt); 483 log = kvasprintf(GFP_KERNEL, fmt, ap); 484 va_end(ap); 485 if (!log) { 486 DRM_ERROR("Cannot allocate memory for the log message.\n"); 487 return -ENOMEM; 488 } 489 490 msg = kasprintf(GFP_KERNEL, "log %s", log); 491 if (!msg) { 492 DRM_ERROR("Cannot allocate memory for host log message.\n"); 493 kfree(log); 494 return -ENOMEM; 495 } 496 497 if (vmw_open_channel(&channel, RPCI_PROTOCOL_NUM)) 498 goto out_open; 499 500 if (vmw_send_msg(&channel, msg)) 501 goto out_msg; 502 503 vmw_close_channel(&channel); 504 kfree(msg); 505 kfree(log); 506 507 return 0; 508 509 out_msg: 510 vmw_close_channel(&channel); 511 out_open: 512 kfree(msg); 513 kfree(log); 514 DRM_ERROR("Failed to send host log message.\n"); 515 516 return -EINVAL; 517 } 518 519 520 /** 521 * vmw_msg_ioctl: Sends and receveives a message to/from host from/to user-space 522 * 523 * Sends a message from user-space to host. 524 * Can also receive a result from host and return that to user-space. 525 * 526 * @dev: Identifies the drm device. 527 * @data: Pointer to the ioctl argument. 528 * @file_priv: Identifies the caller. 529 * Return: Zero on success, negative error code on error. 530 */ 531 532 int vmw_msg_ioctl(struct drm_device *dev, void *data, 533 struct drm_file *file_priv) 534 { 535 struct drm_vmw_msg_arg *arg = 536 (struct drm_vmw_msg_arg *)data; 537 struct rpc_channel channel; 538 char *msg; 539 int length; 540 541 msg = kmalloc(MAX_USER_MSG_LENGTH, GFP_KERNEL); 542 if (!msg) { 543 DRM_ERROR("Cannot allocate memory for log message.\n"); 544 return -ENOMEM; 545 } 546 547 length = strncpy_from_user(msg, (void __user *)((unsigned long)arg->send), 548 MAX_USER_MSG_LENGTH); 549 if (length < 0 || length >= MAX_USER_MSG_LENGTH) { 550 DRM_ERROR("Userspace message access failure.\n"); 551 kfree(msg); 552 return -EINVAL; 553 } 554 555 556 if (vmw_open_channel(&channel, RPCI_PROTOCOL_NUM)) { 557 DRM_ERROR("Failed to open channel.\n"); 558 goto out_open; 559 } 560 561 if (vmw_send_msg(&channel, msg)) { 562 DRM_ERROR("Failed to send message to host.\n"); 563 goto out_msg; 564 } 565 566 if (!arg->send_only) { 567 char *reply = NULL; 568 size_t reply_len = 0; 569 570 if (vmw_recv_msg(&channel, (void *) &reply, &reply_len)) { 571 DRM_ERROR("Failed to receive message from host.\n"); 572 goto out_msg; 573 } 574 if (reply && reply_len > 0) { 575 if (copy_to_user((void __user *)((unsigned long)arg->receive), 576 reply, reply_len)) { 577 DRM_ERROR("Failed to copy message to userspace.\n"); 578 kfree(reply); 579 goto out_msg; 580 } 581 arg->receive_len = (__u32)reply_len; 582 } 583 kfree(reply); 584 } 585 586 vmw_close_channel(&channel); 587 kfree(msg); 588 589 return 0; 590 591 out_msg: 592 vmw_close_channel(&channel); 593 out_open: 594 kfree(msg); 595 596 return -EINVAL; 597 } 598 599 /** 600 * reset_ppn_array: Resets a PPN64 array to INVALID_PPN64 content 601 * 602 * @arr: Array to reset. 603 * @size: Array length. 604 */ 605 static inline void reset_ppn_array(PPN64 *arr, size_t size) 606 { 607 size_t i; 608 609 BUG_ON(!arr || size == 0); 610 611 for (i = 0; i < size; ++i) 612 arr[i] = INVALID_PPN64; 613 } 614 615 /** 616 * hypervisor_ppn_reset_all: Removes all mksGuestStat instance descriptors from 617 * the hypervisor. All related pages should be subsequently unpinned or freed. 618 * 619 */ 620 static inline void hypervisor_ppn_reset_all(void) 621 { 622 vmware_hypercall1(VMW_PORT_CMD_MKSGS_RESET, 0); 623 } 624 625 /** 626 * hypervisor_ppn_add: Adds a single mksGuestStat instance descriptor to the 627 * hypervisor. Any related userspace pages should be pinned in advance. 628 * 629 * @pfn: Physical page number of the instance descriptor 630 */ 631 static inline void hypervisor_ppn_add(PPN64 pfn) 632 { 633 vmware_hypercall1(VMW_PORT_CMD_MKSGS_ADD_PPN, (unsigned long)pfn); 634 } 635 636 /** 637 * hypervisor_ppn_remove: Removes a single mksGuestStat instance descriptor from 638 * the hypervisor. All related pages should be subsequently unpinned or freed. 639 * 640 * @pfn: Physical page number of the instance descriptor 641 */ 642 static inline void hypervisor_ppn_remove(PPN64 pfn) 643 { 644 vmware_hypercall1(VMW_PORT_CMD_MKSGS_REMOVE_PPN, (unsigned long)pfn); 645 } 646 647 #if IS_ENABLED(CONFIG_DRM_VMWGFX_MKSSTATS) 648 649 /* Order of the total number of pages used for kernel-internal mksGuestStat; at least 2 */ 650 #define MKSSTAT_KERNEL_PAGES_ORDER 2 651 /* Header to the text description of mksGuestStat instance descriptor */ 652 #define MKSSTAT_KERNEL_DESCRIPTION "vmwgfx" 653 654 /** 655 * mksstat_init_record_time: Initializes an MKSGuestStatCounterTime-based record 656 * for the respective mksGuestStat index. 657 * 658 * @stat_idx: Index of the MKSGuestStatCounterTime-based mksGuestStat record. 659 * @pstat: Pointer to array of MKSGuestStatCounterTime. 660 * @pinfo: Pointer to array of MKSGuestStatInfoEntry. 661 * @pstrs: Pointer to current end of the name/description sequence. 662 * Return: Pointer to the new end of the names/description sequence. 663 */ 664 665 static inline char *mksstat_init_record_time(mksstat_kern_stats_t stat_idx, 666 MKSGuestStatCounterTime *pstat, MKSGuestStatInfoEntry *pinfo, char *pstrs) 667 { 668 char *const pstrd = pstrs + strlen(mksstat_kern_name_desc[stat_idx][0]) + 1; 669 strcpy(pstrs, mksstat_kern_name_desc[stat_idx][0]); 670 strcpy(pstrd, mksstat_kern_name_desc[stat_idx][1]); 671 672 pinfo[stat_idx].name.s = pstrs; 673 pinfo[stat_idx].description.s = pstrd; 674 pinfo[stat_idx].flags = MKS_GUEST_STAT_FLAG_TIME; 675 pinfo[stat_idx].stat.counterTime = &pstat[stat_idx]; 676 677 return pstrd + strlen(mksstat_kern_name_desc[stat_idx][1]) + 1; 678 } 679 680 /** 681 * mksstat_init_kern_id: Creates a single mksGuestStat instance descriptor and 682 * kernel-internal counters. Adds PFN mapping to the hypervisor. 683 * 684 * Create a single mksGuestStat instance descriptor and corresponding structures 685 * for all kernel-internal counters. The corresponding PFNs are mapped with the 686 * hypervisor. 687 * 688 * @ppage: Output pointer to page containing the instance descriptor. 689 * Return: Zero on success, negative error code on error. 690 */ 691 692 static int mksstat_init_kern_id(struct page **ppage) 693 { 694 MKSGuestStatInstanceDescriptor *pdesc; 695 MKSGuestStatCounterTime *pstat; 696 MKSGuestStatInfoEntry *pinfo; 697 char *pstrs, *pstrs_acc; 698 699 /* Allocate pages for the kernel-internal instance descriptor */ 700 struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, MKSSTAT_KERNEL_PAGES_ORDER); 701 702 if (!page) 703 return -ENOMEM; 704 705 pdesc = page_address(page); 706 pstat = vmw_mksstat_get_kern_pstat(pdesc); 707 pinfo = vmw_mksstat_get_kern_pinfo(pdesc); 708 pstrs = vmw_mksstat_get_kern_pstrs(pdesc); 709 710 /* Set up all kernel-internal counters and corresponding structures */ 711 pstrs_acc = pstrs; 712 pstrs_acc = mksstat_init_record_time(MKSSTAT_KERN_EXECBUF, pstat, pinfo, pstrs_acc); 713 pstrs_acc = mksstat_init_record_time(MKSSTAT_KERN_COTABLE_RESIZE, pstat, pinfo, pstrs_acc); 714 715 /* Add new counters above, in their order of appearance in mksstat_kern_stats_t */ 716 717 BUG_ON(pstrs_acc - pstrs > PAGE_SIZE); 718 719 /* Set up the kernel-internal instance descriptor */ 720 pdesc->reservedMBZ = 0; 721 pdesc->statStartVA = (uintptr_t)pstat; 722 pdesc->strsStartVA = (uintptr_t)pstrs; 723 pdesc->statLength = sizeof(*pstat) * MKSSTAT_KERN_COUNT; 724 pdesc->infoLength = sizeof(*pinfo) * MKSSTAT_KERN_COUNT; 725 pdesc->strsLength = pstrs_acc - pstrs; 726 snprintf(pdesc->description, ARRAY_SIZE(pdesc->description) - 1, "%s pid=%d", 727 MKSSTAT_KERNEL_DESCRIPTION, current->pid); 728 729 pdesc->statPPNs[0] = page_to_pfn(virt_to_page(pstat)); 730 reset_ppn_array(pdesc->statPPNs + 1, ARRAY_SIZE(pdesc->statPPNs) - 1); 731 732 pdesc->infoPPNs[0] = page_to_pfn(virt_to_page(pinfo)); 733 reset_ppn_array(pdesc->infoPPNs + 1, ARRAY_SIZE(pdesc->infoPPNs) - 1); 734 735 pdesc->strsPPNs[0] = page_to_pfn(virt_to_page(pstrs)); 736 reset_ppn_array(pdesc->strsPPNs + 1, ARRAY_SIZE(pdesc->strsPPNs) - 1); 737 738 *ppage = page; 739 740 hypervisor_ppn_add((PPN64)page_to_pfn(page)); 741 742 return 0; 743 } 744 745 /** 746 * vmw_mksstat_get_kern_slot: Acquires a slot for a single kernel-internal 747 * mksGuestStat instance descriptor. 748 * 749 * Find a slot for a single kernel-internal mksGuestStat instance descriptor. 750 * In case no such was already present, allocate a new one and set up a kernel- 751 * internal mksGuestStat instance descriptor for the former. 752 * 753 * @pid: Process for which a slot is sought. 754 * @dev_priv: Identifies the drm private device. 755 * Return: Non-negative slot on success, negative error code on error. 756 */ 757 758 int vmw_mksstat_get_kern_slot(pid_t pid, struct vmw_private *dev_priv) 759 { 760 const size_t base = (u32)hash_32(pid, MKSSTAT_CAPACITY_LOG2); 761 size_t i; 762 763 for (i = 0; i < ARRAY_SIZE(dev_priv->mksstat_kern_pids); ++i) { 764 const size_t slot = (i + base) % ARRAY_SIZE(dev_priv->mksstat_kern_pids); 765 766 /* Check if an instance descriptor for this pid is already present */ 767 if (pid == (pid_t)atomic_read(&dev_priv->mksstat_kern_pids[slot])) 768 return (int)slot; 769 770 /* Set up a new instance descriptor for this pid */ 771 if (!atomic_cmpxchg(&dev_priv->mksstat_kern_pids[slot], 0, MKSSTAT_PID_RESERVED)) { 772 const int ret = mksstat_init_kern_id(&dev_priv->mksstat_kern_pages[slot]); 773 774 if (!ret) { 775 /* Reset top-timer tracking for this slot */ 776 dev_priv->mksstat_kern_top_timer[slot] = MKSSTAT_KERN_COUNT; 777 778 atomic_set(&dev_priv->mksstat_kern_pids[slot], pid); 779 return (int)slot; 780 } 781 782 atomic_set(&dev_priv->mksstat_kern_pids[slot], 0); 783 return ret; 784 } 785 } 786 787 return -ENOSPC; 788 } 789 790 #endif 791 792 /** 793 * vmw_mksstat_cleanup_descriptor: Frees a single userspace-originating 794 * mksGuestStat instance-descriptor page and unpins all related user pages. 795 * 796 * Unpin all user pages realated to this instance descriptor and free 797 * the instance-descriptor page itself. 798 * 799 * @page: Page of the instance descriptor. 800 */ 801 802 static void vmw_mksstat_cleanup_descriptor(struct page *page) 803 { 804 MKSGuestStatInstanceDescriptor *pdesc = page_address(page); 805 size_t i; 806 807 for (i = 0; i < ARRAY_SIZE(pdesc->statPPNs) && pdesc->statPPNs[i] != INVALID_PPN64; ++i) 808 unpin_user_page(pfn_to_page(pdesc->statPPNs[i])); 809 810 for (i = 0; i < ARRAY_SIZE(pdesc->infoPPNs) && pdesc->infoPPNs[i] != INVALID_PPN64; ++i) 811 unpin_user_page(pfn_to_page(pdesc->infoPPNs[i])); 812 813 for (i = 0; i < ARRAY_SIZE(pdesc->strsPPNs) && pdesc->strsPPNs[i] != INVALID_PPN64; ++i) 814 unpin_user_page(pfn_to_page(pdesc->strsPPNs[i])); 815 816 __free_page(page); 817 } 818 819 /** 820 * vmw_mksstat_remove_all: Resets all mksGuestStat instance descriptors 821 * from the hypervisor. 822 * 823 * Discard all hypervisor PFN mappings, containing active mksGuestState instance 824 * descriptors, unpin the related userspace pages and free the related kernel pages. 825 * 826 * @dev_priv: Identifies the drm private device. 827 * Return: Zero on success, negative error code on error. 828 */ 829 830 int vmw_mksstat_remove_all(struct vmw_private *dev_priv) 831 { 832 int ret = 0; 833 size_t i; 834 835 /* Discard all PFN mappings with the hypervisor */ 836 hypervisor_ppn_reset_all(); 837 838 /* Discard all userspace-originating instance descriptors and unpin all related pages */ 839 for (i = 0; i < ARRAY_SIZE(dev_priv->mksstat_user_pids); ++i) { 840 const pid_t pid0 = (pid_t)atomic_read(&dev_priv->mksstat_user_pids[i]); 841 842 if (!pid0) 843 continue; 844 845 if (pid0 != MKSSTAT_PID_RESERVED) { 846 const pid_t pid1 = atomic_cmpxchg(&dev_priv->mksstat_user_pids[i], pid0, MKSSTAT_PID_RESERVED); 847 848 if (!pid1) 849 continue; 850 851 if (pid1 == pid0) { 852 struct page *const page = dev_priv->mksstat_user_pages[i]; 853 854 BUG_ON(!page); 855 856 dev_priv->mksstat_user_pages[i] = NULL; 857 atomic_set(&dev_priv->mksstat_user_pids[i], 0); 858 859 vmw_mksstat_cleanup_descriptor(page); 860 continue; 861 } 862 } 863 864 ret = -EAGAIN; 865 } 866 867 #if IS_ENABLED(CONFIG_DRM_VMWGFX_MKSSTATS) 868 /* Discard all kernel-internal instance descriptors and free all related pages */ 869 for (i = 0; i < ARRAY_SIZE(dev_priv->mksstat_kern_pids); ++i) { 870 const pid_t pid0 = (pid_t)atomic_read(&dev_priv->mksstat_kern_pids[i]); 871 872 if (!pid0) 873 continue; 874 875 if (pid0 != MKSSTAT_PID_RESERVED) { 876 const pid_t pid1 = atomic_cmpxchg(&dev_priv->mksstat_kern_pids[i], pid0, MKSSTAT_PID_RESERVED); 877 878 if (!pid1) 879 continue; 880 881 if (pid1 == pid0) { 882 struct page *const page = dev_priv->mksstat_kern_pages[i]; 883 884 BUG_ON(!page); 885 886 dev_priv->mksstat_kern_pages[i] = NULL; 887 atomic_set(&dev_priv->mksstat_kern_pids[i], 0); 888 889 __free_pages(page, MKSSTAT_KERNEL_PAGES_ORDER); 890 continue; 891 } 892 } 893 894 ret = -EAGAIN; 895 } 896 897 #endif 898 return ret; 899 } 900 901 /** 902 * vmw_mksstat_reset_ioctl: Resets all mksGuestStat instance descriptors 903 * from the hypervisor. 904 * 905 * Discard all hypervisor PFN mappings, containing active mksGuestStat instance 906 * descriptors, unpin the related userspace pages and free the related kernel pages. 907 * 908 * @dev: Identifies the drm device. 909 * @data: Pointer to the ioctl argument. 910 * @file_priv: Identifies the caller; unused. 911 * Return: Zero on success, negative error code on error. 912 */ 913 914 int vmw_mksstat_reset_ioctl(struct drm_device *dev, void *data, 915 struct drm_file *file_priv) 916 { 917 struct vmw_private *const dev_priv = vmw_priv(dev); 918 return vmw_mksstat_remove_all(dev_priv); 919 } 920 921 /** 922 * vmw_mksstat_add_ioctl: Creates a single userspace-originating mksGuestStat 923 * instance descriptor and registers that with the hypervisor. 924 * 925 * Create a hypervisor PFN mapping, containing a single mksGuestStat instance 926 * descriptor and pin the corresponding userspace pages. 927 * 928 * @dev: Identifies the drm device. 929 * @data: Pointer to the ioctl argument. 930 * @file_priv: Identifies the caller; unused. 931 * Return: Zero on success, negative error code on error. 932 */ 933 934 int vmw_mksstat_add_ioctl(struct drm_device *dev, void *data, 935 struct drm_file *file_priv) 936 { 937 struct drm_vmw_mksstat_add_arg *arg = 938 (struct drm_vmw_mksstat_add_arg *) data; 939 940 struct vmw_private *const dev_priv = vmw_priv(dev); 941 942 const size_t num_pages_stat = PFN_UP(arg->stat_len); 943 const size_t num_pages_info = PFN_UP(arg->info_len); 944 const size_t num_pages_strs = PFN_UP(arg->strs_len); 945 long desc_len; 946 long nr_pinned_stat; 947 long nr_pinned_info; 948 long nr_pinned_strs; 949 MKSGuestStatInstanceDescriptor *pdesc; 950 struct page *page = NULL; 951 struct page **pages_stat = NULL; 952 struct page **pages_info = NULL; 953 struct page **pages_strs = NULL; 954 size_t i, slot; 955 int ret_err = -ENOMEM; 956 957 arg->id = -1; 958 959 if (!arg->stat || !arg->info || !arg->strs) 960 return -EINVAL; 961 962 if (!arg->stat_len || !arg->info_len || !arg->strs_len) 963 return -EINVAL; 964 965 if (!arg->description) 966 return -EINVAL; 967 968 if (num_pages_stat > ARRAY_SIZE(pdesc->statPPNs) || 969 num_pages_info > ARRAY_SIZE(pdesc->infoPPNs) || 970 num_pages_strs > ARRAY_SIZE(pdesc->strsPPNs)) 971 return -EINVAL; 972 973 /* Find an available slot in the mksGuestStats user array and reserve it */ 974 for (slot = 0; slot < ARRAY_SIZE(dev_priv->mksstat_user_pids); ++slot) 975 if (!atomic_cmpxchg(&dev_priv->mksstat_user_pids[slot], 0, MKSSTAT_PID_RESERVED)) 976 break; 977 978 if (slot == ARRAY_SIZE(dev_priv->mksstat_user_pids)) 979 return -ENOSPC; 980 981 BUG_ON(dev_priv->mksstat_user_pages[slot]); 982 983 /* Allocate statically-sized temp arrays for pages -- too big to keep in frame */ 984 pages_stat = (struct page **)kmalloc_array( 985 ARRAY_SIZE(pdesc->statPPNs) + 986 ARRAY_SIZE(pdesc->infoPPNs) + 987 ARRAY_SIZE(pdesc->strsPPNs), sizeof(*pages_stat), GFP_KERNEL); 988 989 if (!pages_stat) 990 goto err_nomem; 991 992 pages_info = pages_stat + ARRAY_SIZE(pdesc->statPPNs); 993 pages_strs = pages_info + ARRAY_SIZE(pdesc->infoPPNs); 994 995 /* Allocate a page for the instance descriptor */ 996 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 997 998 if (!page) 999 goto err_nomem; 1000 1001 /* Set up the instance descriptor */ 1002 pdesc = page_address(page); 1003 1004 pdesc->reservedMBZ = 0; 1005 pdesc->statStartVA = arg->stat; 1006 pdesc->strsStartVA = arg->strs; 1007 pdesc->statLength = arg->stat_len; 1008 pdesc->infoLength = arg->info_len; 1009 pdesc->strsLength = arg->strs_len; 1010 desc_len = strncpy_from_user(pdesc->description, u64_to_user_ptr(arg->description), 1011 ARRAY_SIZE(pdesc->description) - 1); 1012 1013 if (desc_len < 0) { 1014 ret_err = -EFAULT; 1015 goto err_nomem; 1016 } 1017 1018 reset_ppn_array(pdesc->statPPNs, ARRAY_SIZE(pdesc->statPPNs)); 1019 reset_ppn_array(pdesc->infoPPNs, ARRAY_SIZE(pdesc->infoPPNs)); 1020 reset_ppn_array(pdesc->strsPPNs, ARRAY_SIZE(pdesc->strsPPNs)); 1021 1022 /* Pin mksGuestStat user pages and store those in the instance descriptor */ 1023 nr_pinned_stat = pin_user_pages_fast(arg->stat, num_pages_stat, FOLL_LONGTERM, pages_stat); 1024 if (num_pages_stat != nr_pinned_stat) 1025 goto err_pin_stat; 1026 1027 for (i = 0; i < num_pages_stat; ++i) 1028 pdesc->statPPNs[i] = page_to_pfn(pages_stat[i]); 1029 1030 nr_pinned_info = pin_user_pages_fast(arg->info, num_pages_info, FOLL_LONGTERM, pages_info); 1031 if (num_pages_info != nr_pinned_info) 1032 goto err_pin_info; 1033 1034 for (i = 0; i < num_pages_info; ++i) 1035 pdesc->infoPPNs[i] = page_to_pfn(pages_info[i]); 1036 1037 nr_pinned_strs = pin_user_pages_fast(arg->strs, num_pages_strs, FOLL_LONGTERM, pages_strs); 1038 if (num_pages_strs != nr_pinned_strs) 1039 goto err_pin_strs; 1040 1041 for (i = 0; i < num_pages_strs; ++i) 1042 pdesc->strsPPNs[i] = page_to_pfn(pages_strs[i]); 1043 1044 /* Send the descriptor to the host via a hypervisor call. The mksGuestStat 1045 pages will remain in use until the user requests a matching remove stats 1046 or a stats reset occurs. */ 1047 hypervisor_ppn_add((PPN64)page_to_pfn(page)); 1048 1049 dev_priv->mksstat_user_pages[slot] = page; 1050 atomic_set(&dev_priv->mksstat_user_pids[slot], task_pgrp_vnr(current)); 1051 1052 arg->id = slot; 1053 1054 DRM_DEV_INFO(dev->dev, "pid=%d arg.description='%.*s' id=%zu\n", current->pid, (int)desc_len, pdesc->description, slot); 1055 1056 kfree(pages_stat); 1057 return 0; 1058 1059 err_pin_strs: 1060 if (nr_pinned_strs > 0) 1061 unpin_user_pages(pages_strs, nr_pinned_strs); 1062 1063 err_pin_info: 1064 if (nr_pinned_info > 0) 1065 unpin_user_pages(pages_info, nr_pinned_info); 1066 1067 err_pin_stat: 1068 if (nr_pinned_stat > 0) 1069 unpin_user_pages(pages_stat, nr_pinned_stat); 1070 1071 err_nomem: 1072 atomic_set(&dev_priv->mksstat_user_pids[slot], 0); 1073 if (page) 1074 __free_page(page); 1075 kfree(pages_stat); 1076 1077 return ret_err; 1078 } 1079 1080 /** 1081 * vmw_mksstat_remove_ioctl: Removes a single userspace-originating mksGuestStat 1082 * instance descriptor from the hypervisor. 1083 * 1084 * Discard a hypervisor PFN mapping, containing a single mksGuestStat instance 1085 * descriptor and unpin the corresponding userspace pages. 1086 * 1087 * @dev: Identifies the drm device. 1088 * @data: Pointer to the ioctl argument. 1089 * @file_priv: Identifies the caller; unused. 1090 * Return: Zero on success, negative error code on error. 1091 */ 1092 1093 int vmw_mksstat_remove_ioctl(struct drm_device *dev, void *data, 1094 struct drm_file *file_priv) 1095 { 1096 struct drm_vmw_mksstat_remove_arg *arg = 1097 (struct drm_vmw_mksstat_remove_arg *) data; 1098 1099 struct vmw_private *const dev_priv = vmw_priv(dev); 1100 1101 const size_t slot = arg->id; 1102 pid_t pgid, pid; 1103 1104 if (slot >= ARRAY_SIZE(dev_priv->mksstat_user_pids)) 1105 return -EINVAL; 1106 1107 DRM_DEV_INFO(dev->dev, "pid=%d arg.id=%zu\n", current->pid, slot); 1108 1109 pgid = task_pgrp_vnr(current); 1110 pid = atomic_cmpxchg(&dev_priv->mksstat_user_pids[slot], pgid, MKSSTAT_PID_RESERVED); 1111 1112 if (!pid) 1113 return 0; 1114 1115 if (pid == pgid) { 1116 struct page *const page = dev_priv->mksstat_user_pages[slot]; 1117 1118 BUG_ON(!page); 1119 1120 dev_priv->mksstat_user_pages[slot] = NULL; 1121 atomic_set(&dev_priv->mksstat_user_pids[slot], 0); 1122 1123 hypervisor_ppn_remove((PPN64)page_to_pfn(page)); 1124 1125 vmw_mksstat_cleanup_descriptor(page); 1126 return 0; 1127 } 1128 1129 return -EAGAIN; 1130 } 1131 1132 /** 1133 * vmw_disable_backdoor: Disables all backdoor communication 1134 * with the hypervisor. 1135 */ 1136 void vmw_disable_backdoor(void) 1137 { 1138 vmw_msg_enabled = 0; 1139 } 1140