1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/device.h> 24 #include <linux/export.h> 25 #include <linux/err.h> 26 #include <linux/fs.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/uaccess.h> 30 #include <linux/compat.h> 31 #include <uapi/linux/kfd_ioctl.h> 32 #include <linux/time.h> 33 #include <linux/mm.h> 34 #include <linux/mman.h> 35 #include <asm/processor.h> 36 #include "kfd_priv.h" 37 #include "kfd_device_queue_manager.h" 38 #include "kfd_dbgmgr.h" 39 40 static long kfd_ioctl(struct file *, unsigned int, unsigned long); 41 static int kfd_open(struct inode *, struct file *); 42 static int kfd_mmap(struct file *, struct vm_area_struct *); 43 44 static const char kfd_dev_name[] = "kfd"; 45 46 static const struct file_operations kfd_fops = { 47 .owner = THIS_MODULE, 48 .unlocked_ioctl = kfd_ioctl, 49 .compat_ioctl = kfd_ioctl, 50 .open = kfd_open, 51 .mmap = kfd_mmap, 52 }; 53 54 static int kfd_char_dev_major = -1; 55 static struct class *kfd_class; 56 struct device *kfd_device; 57 58 int kfd_chardev_init(void) 59 { 60 int err = 0; 61 62 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); 63 err = kfd_char_dev_major; 64 if (err < 0) 65 goto err_register_chrdev; 66 67 kfd_class = class_create(THIS_MODULE, kfd_dev_name); 68 err = PTR_ERR(kfd_class); 69 if (IS_ERR(kfd_class)) 70 goto err_class_create; 71 72 kfd_device = device_create(kfd_class, NULL, 73 MKDEV(kfd_char_dev_major, 0), 74 NULL, kfd_dev_name); 75 err = PTR_ERR(kfd_device); 76 if (IS_ERR(kfd_device)) 77 goto err_device_create; 78 79 return 0; 80 81 err_device_create: 82 class_destroy(kfd_class); 83 err_class_create: 84 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 85 err_register_chrdev: 86 return err; 87 } 88 89 void kfd_chardev_exit(void) 90 { 91 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); 92 class_destroy(kfd_class); 93 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 94 } 95 96 struct device *kfd_chardev(void) 97 { 98 return kfd_device; 99 } 100 101 102 static int kfd_open(struct inode *inode, struct file *filep) 103 { 104 struct kfd_process *process; 105 bool is_32bit_user_mode; 106 107 if (iminor(inode) != 0) 108 return -ENODEV; 109 110 is_32bit_user_mode = in_compat_syscall(); 111 112 if (is_32bit_user_mode) { 113 dev_warn(kfd_device, 114 "Process %d (32-bit) failed to open /dev/kfd\n" 115 "32-bit processes are not supported by amdkfd\n", 116 current->pid); 117 return -EPERM; 118 } 119 120 process = kfd_create_process(filep); 121 if (IS_ERR(process)) 122 return PTR_ERR(process); 123 124 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", 125 process->pasid, process->is_32bit_user_mode); 126 127 return 0; 128 } 129 130 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 131 void *data) 132 { 133 struct kfd_ioctl_get_version_args *args = data; 134 135 args->major_version = KFD_IOCTL_MAJOR_VERSION; 136 args->minor_version = KFD_IOCTL_MINOR_VERSION; 137 138 return 0; 139 } 140 141 static int set_queue_properties_from_user(struct queue_properties *q_properties, 142 struct kfd_ioctl_create_queue_args *args) 143 { 144 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 145 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 146 return -EINVAL; 147 } 148 149 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 150 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 151 return -EINVAL; 152 } 153 154 if ((args->ring_base_address) && 155 (!access_ok(VERIFY_WRITE, 156 (const void __user *) args->ring_base_address, 157 sizeof(uint64_t)))) { 158 pr_err("Can't access ring base address\n"); 159 return -EFAULT; 160 } 161 162 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 163 pr_err("Ring size must be a power of 2 or 0\n"); 164 return -EINVAL; 165 } 166 167 if (!access_ok(VERIFY_WRITE, 168 (const void __user *) args->read_pointer_address, 169 sizeof(uint32_t))) { 170 pr_err("Can't access read pointer\n"); 171 return -EFAULT; 172 } 173 174 if (!access_ok(VERIFY_WRITE, 175 (const void __user *) args->write_pointer_address, 176 sizeof(uint32_t))) { 177 pr_err("Can't access write pointer\n"); 178 return -EFAULT; 179 } 180 181 if (args->eop_buffer_address && 182 !access_ok(VERIFY_WRITE, 183 (const void __user *) args->eop_buffer_address, 184 sizeof(uint32_t))) { 185 pr_debug("Can't access eop buffer"); 186 return -EFAULT; 187 } 188 189 if (args->ctx_save_restore_address && 190 !access_ok(VERIFY_WRITE, 191 (const void __user *) args->ctx_save_restore_address, 192 sizeof(uint32_t))) { 193 pr_debug("Can't access ctx save restore buffer"); 194 return -EFAULT; 195 } 196 197 q_properties->is_interop = false; 198 q_properties->queue_percent = args->queue_percentage; 199 q_properties->priority = args->queue_priority; 200 q_properties->queue_address = args->ring_base_address; 201 q_properties->queue_size = args->ring_size; 202 q_properties->read_ptr = (uint32_t *) args->read_pointer_address; 203 q_properties->write_ptr = (uint32_t *) args->write_pointer_address; 204 q_properties->eop_ring_buffer_address = args->eop_buffer_address; 205 q_properties->eop_ring_buffer_size = args->eop_buffer_size; 206 q_properties->ctx_save_restore_area_address = 207 args->ctx_save_restore_address; 208 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; 209 q_properties->ctl_stack_size = args->ctl_stack_size; 210 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || 211 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 212 q_properties->type = KFD_QUEUE_TYPE_COMPUTE; 213 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) 214 q_properties->type = KFD_QUEUE_TYPE_SDMA; 215 else 216 return -ENOTSUPP; 217 218 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 219 q_properties->format = KFD_QUEUE_FORMAT_AQL; 220 else 221 q_properties->format = KFD_QUEUE_FORMAT_PM4; 222 223 pr_debug("Queue Percentage: %d, %d\n", 224 q_properties->queue_percent, args->queue_percentage); 225 226 pr_debug("Queue Priority: %d, %d\n", 227 q_properties->priority, args->queue_priority); 228 229 pr_debug("Queue Address: 0x%llX, 0x%llX\n", 230 q_properties->queue_address, args->ring_base_address); 231 232 pr_debug("Queue Size: 0x%llX, %u\n", 233 q_properties->queue_size, args->ring_size); 234 235 pr_debug("Queue r/w Pointers: %p, %p\n", 236 q_properties->read_ptr, 237 q_properties->write_ptr); 238 239 pr_debug("Queue Format: %d\n", q_properties->format); 240 241 pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address); 242 243 pr_debug("Queue CTX save area: 0x%llX\n", 244 q_properties->ctx_save_restore_area_address); 245 246 return 0; 247 } 248 249 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 250 void *data) 251 { 252 struct kfd_ioctl_create_queue_args *args = data; 253 struct kfd_dev *dev; 254 int err = 0; 255 unsigned int queue_id; 256 struct kfd_process_device *pdd; 257 struct queue_properties q_properties; 258 259 memset(&q_properties, 0, sizeof(struct queue_properties)); 260 261 pr_debug("Creating queue ioctl\n"); 262 263 err = set_queue_properties_from_user(&q_properties, args); 264 if (err) 265 return err; 266 267 pr_debug("Looking for gpu id 0x%x\n", args->gpu_id); 268 dev = kfd_device_by_id(args->gpu_id); 269 if (!dev) { 270 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id); 271 return -EINVAL; 272 } 273 274 mutex_lock(&p->mutex); 275 276 pdd = kfd_bind_process_to_device(dev, p); 277 if (IS_ERR(pdd)) { 278 err = -ESRCH; 279 goto err_bind_process; 280 } 281 282 pr_debug("Creating queue for PASID %d on gpu 0x%x\n", 283 p->pasid, 284 dev->id); 285 286 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id); 287 if (err != 0) 288 goto err_create_queue; 289 290 args->queue_id = queue_id; 291 292 293 /* Return gpu_id as doorbell offset for mmap usage */ 294 args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id); 295 args->doorbell_offset <<= PAGE_SHIFT; 296 297 mutex_unlock(&p->mutex); 298 299 pr_debug("Queue id %d was created successfully\n", args->queue_id); 300 301 pr_debug("Ring buffer address == 0x%016llX\n", 302 args->ring_base_address); 303 304 pr_debug("Read ptr address == 0x%016llX\n", 305 args->read_pointer_address); 306 307 pr_debug("Write ptr address == 0x%016llX\n", 308 args->write_pointer_address); 309 310 return 0; 311 312 err_create_queue: 313 err_bind_process: 314 mutex_unlock(&p->mutex); 315 return err; 316 } 317 318 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 319 void *data) 320 { 321 int retval; 322 struct kfd_ioctl_destroy_queue_args *args = data; 323 324 pr_debug("Destroying queue id %d for pasid %d\n", 325 args->queue_id, 326 p->pasid); 327 328 mutex_lock(&p->mutex); 329 330 retval = pqm_destroy_queue(&p->pqm, args->queue_id); 331 332 mutex_unlock(&p->mutex); 333 return retval; 334 } 335 336 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 337 void *data) 338 { 339 int retval; 340 struct kfd_ioctl_update_queue_args *args = data; 341 struct queue_properties properties; 342 343 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 344 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 345 return -EINVAL; 346 } 347 348 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 349 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 350 return -EINVAL; 351 } 352 353 if ((args->ring_base_address) && 354 (!access_ok(VERIFY_WRITE, 355 (const void __user *) args->ring_base_address, 356 sizeof(uint64_t)))) { 357 pr_err("Can't access ring base address\n"); 358 return -EFAULT; 359 } 360 361 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 362 pr_err("Ring size must be a power of 2 or 0\n"); 363 return -EINVAL; 364 } 365 366 properties.queue_address = args->ring_base_address; 367 properties.queue_size = args->ring_size; 368 properties.queue_percent = args->queue_percentage; 369 properties.priority = args->queue_priority; 370 371 pr_debug("Updating queue id %d for pasid %d\n", 372 args->queue_id, p->pasid); 373 374 mutex_lock(&p->mutex); 375 376 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); 377 378 mutex_unlock(&p->mutex); 379 380 return retval; 381 } 382 383 static int kfd_ioctl_set_memory_policy(struct file *filep, 384 struct kfd_process *p, void *data) 385 { 386 struct kfd_ioctl_set_memory_policy_args *args = data; 387 struct kfd_dev *dev; 388 int err = 0; 389 struct kfd_process_device *pdd; 390 enum cache_policy default_policy, alternate_policy; 391 392 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT 393 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 394 return -EINVAL; 395 } 396 397 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 398 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 399 return -EINVAL; 400 } 401 402 dev = kfd_device_by_id(args->gpu_id); 403 if (!dev) 404 return -EINVAL; 405 406 mutex_lock(&p->mutex); 407 408 pdd = kfd_bind_process_to_device(dev, p); 409 if (IS_ERR(pdd)) { 410 err = -ESRCH; 411 goto out; 412 } 413 414 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 415 ? cache_policy_coherent : cache_policy_noncoherent; 416 417 alternate_policy = 418 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 419 ? cache_policy_coherent : cache_policy_noncoherent; 420 421 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, 422 &pdd->qpd, 423 default_policy, 424 alternate_policy, 425 (void __user *)args->alternate_aperture_base, 426 args->alternate_aperture_size)) 427 err = -EINVAL; 428 429 out: 430 mutex_unlock(&p->mutex); 431 432 return err; 433 } 434 435 static int kfd_ioctl_set_trap_handler(struct file *filep, 436 struct kfd_process *p, void *data) 437 { 438 struct kfd_ioctl_set_trap_handler_args *args = data; 439 struct kfd_dev *dev; 440 int err = 0; 441 struct kfd_process_device *pdd; 442 443 dev = kfd_device_by_id(args->gpu_id); 444 if (dev == NULL) 445 return -EINVAL; 446 447 mutex_lock(&p->mutex); 448 449 pdd = kfd_bind_process_to_device(dev, p); 450 if (IS_ERR(pdd)) { 451 err = -ESRCH; 452 goto out; 453 } 454 455 if (dev->dqm->ops.set_trap_handler(dev->dqm, 456 &pdd->qpd, 457 args->tba_addr, 458 args->tma_addr)) 459 err = -EINVAL; 460 461 out: 462 mutex_unlock(&p->mutex); 463 464 return err; 465 } 466 467 static int kfd_ioctl_dbg_register(struct file *filep, 468 struct kfd_process *p, void *data) 469 { 470 struct kfd_ioctl_dbg_register_args *args = data; 471 struct kfd_dev *dev; 472 struct kfd_dbgmgr *dbgmgr_ptr; 473 struct kfd_process_device *pdd; 474 bool create_ok; 475 long status = 0; 476 477 dev = kfd_device_by_id(args->gpu_id); 478 if (!dev) 479 return -EINVAL; 480 481 if (dev->device_info->asic_family == CHIP_CARRIZO) { 482 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); 483 return -EINVAL; 484 } 485 486 mutex_lock(&p->mutex); 487 mutex_lock(kfd_get_dbgmgr_mutex()); 488 489 /* 490 * make sure that we have pdd, if this the first queue created for 491 * this process 492 */ 493 pdd = kfd_bind_process_to_device(dev, p); 494 if (IS_ERR(pdd)) { 495 status = PTR_ERR(pdd); 496 goto out; 497 } 498 499 if (!dev->dbgmgr) { 500 /* In case of a legal call, we have no dbgmgr yet */ 501 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); 502 if (create_ok) { 503 status = kfd_dbgmgr_register(dbgmgr_ptr, p); 504 if (status != 0) 505 kfd_dbgmgr_destroy(dbgmgr_ptr); 506 else 507 dev->dbgmgr = dbgmgr_ptr; 508 } 509 } else { 510 pr_debug("debugger already registered\n"); 511 status = -EINVAL; 512 } 513 514 out: 515 mutex_unlock(kfd_get_dbgmgr_mutex()); 516 mutex_unlock(&p->mutex); 517 518 return status; 519 } 520 521 static int kfd_ioctl_dbg_unregister(struct file *filep, 522 struct kfd_process *p, void *data) 523 { 524 struct kfd_ioctl_dbg_unregister_args *args = data; 525 struct kfd_dev *dev; 526 long status; 527 528 dev = kfd_device_by_id(args->gpu_id); 529 if (!dev || !dev->dbgmgr) 530 return -EINVAL; 531 532 if (dev->device_info->asic_family == CHIP_CARRIZO) { 533 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n"); 534 return -EINVAL; 535 } 536 537 mutex_lock(kfd_get_dbgmgr_mutex()); 538 539 status = kfd_dbgmgr_unregister(dev->dbgmgr, p); 540 if (!status) { 541 kfd_dbgmgr_destroy(dev->dbgmgr); 542 dev->dbgmgr = NULL; 543 } 544 545 mutex_unlock(kfd_get_dbgmgr_mutex()); 546 547 return status; 548 } 549 550 /* 551 * Parse and generate variable size data structure for address watch. 552 * Total size of the buffer and # watch points is limited in order 553 * to prevent kernel abuse. (no bearing to the much smaller HW limitation 554 * which is enforced by dbgdev module) 555 * please also note that the watch address itself are not "copied from user", 556 * since it be set into the HW in user mode values. 557 * 558 */ 559 static int kfd_ioctl_dbg_address_watch(struct file *filep, 560 struct kfd_process *p, void *data) 561 { 562 struct kfd_ioctl_dbg_address_watch_args *args = data; 563 struct kfd_dev *dev; 564 struct dbg_address_watch_info aw_info; 565 unsigned char *args_buff; 566 long status; 567 void __user *cmd_from_user; 568 uint64_t watch_mask_value = 0; 569 unsigned int args_idx = 0; 570 571 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); 572 573 dev = kfd_device_by_id(args->gpu_id); 574 if (!dev) 575 return -EINVAL; 576 577 if (dev->device_info->asic_family == CHIP_CARRIZO) { 578 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 579 return -EINVAL; 580 } 581 582 cmd_from_user = (void __user *) args->content_ptr; 583 584 /* Validate arguments */ 585 586 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || 587 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || 588 (cmd_from_user == NULL)) 589 return -EINVAL; 590 591 /* this is the actual buffer to work with */ 592 args_buff = memdup_user(cmd_from_user, 593 args->buf_size_in_bytes - sizeof(*args)); 594 if (IS_ERR(args_buff)) 595 return PTR_ERR(args_buff); 596 597 aw_info.process = p; 598 599 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); 600 args_idx += sizeof(aw_info.num_watch_points); 601 602 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; 603 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; 604 605 /* 606 * set watch address base pointer to point on the array base 607 * within args_buff 608 */ 609 aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; 610 611 /* skip over the addresses buffer */ 612 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; 613 614 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { 615 status = -EINVAL; 616 goto out; 617 } 618 619 watch_mask_value = (uint64_t) args_buff[args_idx]; 620 621 if (watch_mask_value > 0) { 622 /* 623 * There is an array of masks. 624 * set watch mask base pointer to point on the array base 625 * within args_buff 626 */ 627 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; 628 629 /* skip over the masks buffer */ 630 args_idx += sizeof(aw_info.watch_mask) * 631 aw_info.num_watch_points; 632 } else { 633 /* just the NULL mask, set to NULL and skip over it */ 634 aw_info.watch_mask = NULL; 635 args_idx += sizeof(aw_info.watch_mask); 636 } 637 638 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { 639 status = -EINVAL; 640 goto out; 641 } 642 643 /* Currently HSA Event is not supported for DBG */ 644 aw_info.watch_event = NULL; 645 646 mutex_lock(kfd_get_dbgmgr_mutex()); 647 648 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); 649 650 mutex_unlock(kfd_get_dbgmgr_mutex()); 651 652 out: 653 kfree(args_buff); 654 655 return status; 656 } 657 658 /* Parse and generate fixed size data structure for wave control */ 659 static int kfd_ioctl_dbg_wave_control(struct file *filep, 660 struct kfd_process *p, void *data) 661 { 662 struct kfd_ioctl_dbg_wave_control_args *args = data; 663 struct kfd_dev *dev; 664 struct dbg_wave_control_info wac_info; 665 unsigned char *args_buff; 666 uint32_t computed_buff_size; 667 long status; 668 void __user *cmd_from_user; 669 unsigned int args_idx = 0; 670 671 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); 672 673 /* we use compact form, independent of the packing attribute value */ 674 computed_buff_size = sizeof(*args) + 675 sizeof(wac_info.mode) + 676 sizeof(wac_info.operand) + 677 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + 678 sizeof(wac_info.dbgWave_msg.MemoryVA) + 679 sizeof(wac_info.trapId); 680 681 dev = kfd_device_by_id(args->gpu_id); 682 if (!dev) 683 return -EINVAL; 684 685 if (dev->device_info->asic_family == CHIP_CARRIZO) { 686 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 687 return -EINVAL; 688 } 689 690 /* input size must match the computed "compact" size */ 691 if (args->buf_size_in_bytes != computed_buff_size) { 692 pr_debug("size mismatch, computed : actual %u : %u\n", 693 args->buf_size_in_bytes, computed_buff_size); 694 return -EINVAL; 695 } 696 697 cmd_from_user = (void __user *) args->content_ptr; 698 699 if (cmd_from_user == NULL) 700 return -EINVAL; 701 702 /* copy the entire buffer from user */ 703 704 args_buff = memdup_user(cmd_from_user, 705 args->buf_size_in_bytes - sizeof(*args)); 706 if (IS_ERR(args_buff)) 707 return PTR_ERR(args_buff); 708 709 /* move ptr to the start of the "pay-load" area */ 710 wac_info.process = p; 711 712 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); 713 args_idx += sizeof(wac_info.operand); 714 715 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); 716 args_idx += sizeof(wac_info.mode); 717 718 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); 719 args_idx += sizeof(wac_info.trapId); 720 721 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 722 *((uint32_t *)(&args_buff[args_idx])); 723 wac_info.dbgWave_msg.MemoryVA = NULL; 724 725 mutex_lock(kfd_get_dbgmgr_mutex()); 726 727 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", 728 wac_info.process, wac_info.operand, 729 wac_info.mode, wac_info.trapId, 730 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); 731 732 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); 733 734 pr_debug("Returned status of dbg manager is %ld\n", status); 735 736 mutex_unlock(kfd_get_dbgmgr_mutex()); 737 738 kfree(args_buff); 739 740 return status; 741 } 742 743 static int kfd_ioctl_get_clock_counters(struct file *filep, 744 struct kfd_process *p, void *data) 745 { 746 struct kfd_ioctl_get_clock_counters_args *args = data; 747 struct kfd_dev *dev; 748 struct timespec64 time; 749 750 dev = kfd_device_by_id(args->gpu_id); 751 if (dev == NULL) 752 return -EINVAL; 753 754 /* Reading GPU clock counter from KGD */ 755 args->gpu_clock_counter = 756 dev->kfd2kgd->get_gpu_clock_counter(dev->kgd); 757 758 /* No access to rdtsc. Using raw monotonic time */ 759 getrawmonotonic64(&time); 760 args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time); 761 762 get_monotonic_boottime64(&time); 763 args->system_clock_counter = (uint64_t)timespec64_to_ns(&time); 764 765 /* Since the counter is in nano-seconds we use 1GHz frequency */ 766 args->system_clock_freq = 1000000000; 767 768 return 0; 769 } 770 771 772 static int kfd_ioctl_get_process_apertures(struct file *filp, 773 struct kfd_process *p, void *data) 774 { 775 struct kfd_ioctl_get_process_apertures_args *args = data; 776 struct kfd_process_device_apertures *pAperture; 777 struct kfd_process_device *pdd; 778 779 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); 780 781 args->num_of_nodes = 0; 782 783 mutex_lock(&p->mutex); 784 785 /*if the process-device list isn't empty*/ 786 if (kfd_has_process_device_data(p)) { 787 /* Run over all pdd of the process */ 788 pdd = kfd_get_first_process_device_data(p); 789 do { 790 pAperture = 791 &args->process_apertures[args->num_of_nodes]; 792 pAperture->gpu_id = pdd->dev->id; 793 pAperture->lds_base = pdd->lds_base; 794 pAperture->lds_limit = pdd->lds_limit; 795 pAperture->gpuvm_base = pdd->gpuvm_base; 796 pAperture->gpuvm_limit = pdd->gpuvm_limit; 797 pAperture->scratch_base = pdd->scratch_base; 798 pAperture->scratch_limit = pdd->scratch_limit; 799 800 dev_dbg(kfd_device, 801 "node id %u\n", args->num_of_nodes); 802 dev_dbg(kfd_device, 803 "gpu id %u\n", pdd->dev->id); 804 dev_dbg(kfd_device, 805 "lds_base %llX\n", pdd->lds_base); 806 dev_dbg(kfd_device, 807 "lds_limit %llX\n", pdd->lds_limit); 808 dev_dbg(kfd_device, 809 "gpuvm_base %llX\n", pdd->gpuvm_base); 810 dev_dbg(kfd_device, 811 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 812 dev_dbg(kfd_device, 813 "scratch_base %llX\n", pdd->scratch_base); 814 dev_dbg(kfd_device, 815 "scratch_limit %llX\n", pdd->scratch_limit); 816 817 args->num_of_nodes++; 818 819 pdd = kfd_get_next_process_device_data(p, pdd); 820 } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 821 } 822 823 mutex_unlock(&p->mutex); 824 825 return 0; 826 } 827 828 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, 829 void *data) 830 { 831 struct kfd_ioctl_create_event_args *args = data; 832 int err; 833 834 err = kfd_event_create(filp, p, args->event_type, 835 args->auto_reset != 0, args->node_id, 836 &args->event_id, &args->event_trigger_data, 837 &args->event_page_offset, 838 &args->event_slot_index); 839 840 return err; 841 } 842 843 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, 844 void *data) 845 { 846 struct kfd_ioctl_destroy_event_args *args = data; 847 848 return kfd_event_destroy(p, args->event_id); 849 } 850 851 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, 852 void *data) 853 { 854 struct kfd_ioctl_set_event_args *args = data; 855 856 return kfd_set_event(p, args->event_id); 857 } 858 859 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, 860 void *data) 861 { 862 struct kfd_ioctl_reset_event_args *args = data; 863 864 return kfd_reset_event(p, args->event_id); 865 } 866 867 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, 868 void *data) 869 { 870 struct kfd_ioctl_wait_events_args *args = data; 871 int err; 872 873 err = kfd_wait_on_events(p, args->num_events, 874 (void __user *)args->events_ptr, 875 (args->wait_for_all != 0), 876 args->timeout, &args->wait_result); 877 878 return err; 879 } 880 static int kfd_ioctl_set_scratch_backing_va(struct file *filep, 881 struct kfd_process *p, void *data) 882 { 883 struct kfd_ioctl_set_scratch_backing_va_args *args = data; 884 struct kfd_process_device *pdd; 885 struct kfd_dev *dev; 886 long err; 887 888 dev = kfd_device_by_id(args->gpu_id); 889 if (!dev) 890 return -EINVAL; 891 892 mutex_lock(&p->mutex); 893 894 pdd = kfd_bind_process_to_device(dev, p); 895 if (IS_ERR(pdd)) { 896 err = PTR_ERR(pdd); 897 goto bind_process_to_device_fail; 898 } 899 900 pdd->qpd.sh_hidden_private_base = args->va_addr; 901 902 mutex_unlock(&p->mutex); 903 904 if (sched_policy == KFD_SCHED_POLICY_NO_HWS && pdd->qpd.vmid != 0) 905 dev->kfd2kgd->set_scratch_backing_va( 906 dev->kgd, args->va_addr, pdd->qpd.vmid); 907 908 return 0; 909 910 bind_process_to_device_fail: 911 mutex_unlock(&p->mutex); 912 return err; 913 } 914 915 static int kfd_ioctl_get_tile_config(struct file *filep, 916 struct kfd_process *p, void *data) 917 { 918 struct kfd_ioctl_get_tile_config_args *args = data; 919 struct kfd_dev *dev; 920 struct tile_config config; 921 int err = 0; 922 923 dev = kfd_device_by_id(args->gpu_id); 924 if (!dev) 925 return -EINVAL; 926 927 dev->kfd2kgd->get_tile_config(dev->kgd, &config); 928 929 args->gb_addr_config = config.gb_addr_config; 930 args->num_banks = config.num_banks; 931 args->num_ranks = config.num_ranks; 932 933 if (args->num_tile_configs > config.num_tile_configs) 934 args->num_tile_configs = config.num_tile_configs; 935 err = copy_to_user((void __user *)args->tile_config_ptr, 936 config.tile_config_ptr, 937 args->num_tile_configs * sizeof(uint32_t)); 938 if (err) { 939 args->num_tile_configs = 0; 940 return -EFAULT; 941 } 942 943 if (args->num_macro_tile_configs > config.num_macro_tile_configs) 944 args->num_macro_tile_configs = 945 config.num_macro_tile_configs; 946 err = copy_to_user((void __user *)args->macro_tile_config_ptr, 947 config.macro_tile_config_ptr, 948 args->num_macro_tile_configs * sizeof(uint32_t)); 949 if (err) { 950 args->num_macro_tile_configs = 0; 951 return -EFAULT; 952 } 953 954 return 0; 955 } 956 957 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ 958 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ 959 .cmd_drv = 0, .name = #ioctl} 960 961 /** Ioctl table */ 962 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { 963 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, 964 kfd_ioctl_get_version, 0), 965 966 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, 967 kfd_ioctl_create_queue, 0), 968 969 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, 970 kfd_ioctl_destroy_queue, 0), 971 972 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, 973 kfd_ioctl_set_memory_policy, 0), 974 975 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, 976 kfd_ioctl_get_clock_counters, 0), 977 978 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, 979 kfd_ioctl_get_process_apertures, 0), 980 981 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, 982 kfd_ioctl_update_queue, 0), 983 984 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, 985 kfd_ioctl_create_event, 0), 986 987 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, 988 kfd_ioctl_destroy_event, 0), 989 990 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, 991 kfd_ioctl_set_event, 0), 992 993 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, 994 kfd_ioctl_reset_event, 0), 995 996 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, 997 kfd_ioctl_wait_events, 0), 998 999 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, 1000 kfd_ioctl_dbg_register, 0), 1001 1002 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, 1003 kfd_ioctl_dbg_unregister, 0), 1004 1005 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, 1006 kfd_ioctl_dbg_address_watch, 0), 1007 1008 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, 1009 kfd_ioctl_dbg_wave_control, 0), 1010 1011 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA, 1012 kfd_ioctl_set_scratch_backing_va, 0), 1013 1014 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG, 1015 kfd_ioctl_get_tile_config, 0), 1016 1017 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER, 1018 kfd_ioctl_set_trap_handler, 0), 1019 }; 1020 1021 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) 1022 1023 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 1024 { 1025 struct kfd_process *process; 1026 amdkfd_ioctl_t *func; 1027 const struct amdkfd_ioctl_desc *ioctl = NULL; 1028 unsigned int nr = _IOC_NR(cmd); 1029 char stack_kdata[128]; 1030 char *kdata = NULL; 1031 unsigned int usize, asize; 1032 int retcode = -EINVAL; 1033 1034 if (nr >= AMDKFD_CORE_IOCTL_COUNT) 1035 goto err_i1; 1036 1037 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { 1038 u32 amdkfd_size; 1039 1040 ioctl = &amdkfd_ioctls[nr]; 1041 1042 amdkfd_size = _IOC_SIZE(ioctl->cmd); 1043 usize = asize = _IOC_SIZE(cmd); 1044 if (amdkfd_size > asize) 1045 asize = amdkfd_size; 1046 1047 cmd = ioctl->cmd; 1048 } else 1049 goto err_i1; 1050 1051 dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); 1052 1053 process = kfd_get_process(current); 1054 if (IS_ERR(process)) { 1055 dev_dbg(kfd_device, "no process\n"); 1056 goto err_i1; 1057 } 1058 1059 /* Do not trust userspace, use our own definition */ 1060 func = ioctl->func; 1061 1062 if (unlikely(!func)) { 1063 dev_dbg(kfd_device, "no function\n"); 1064 retcode = -EINVAL; 1065 goto err_i1; 1066 } 1067 1068 if (cmd & (IOC_IN | IOC_OUT)) { 1069 if (asize <= sizeof(stack_kdata)) { 1070 kdata = stack_kdata; 1071 } else { 1072 kdata = kmalloc(asize, GFP_KERNEL); 1073 if (!kdata) { 1074 retcode = -ENOMEM; 1075 goto err_i1; 1076 } 1077 } 1078 if (asize > usize) 1079 memset(kdata + usize, 0, asize - usize); 1080 } 1081 1082 if (cmd & IOC_IN) { 1083 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { 1084 retcode = -EFAULT; 1085 goto err_i1; 1086 } 1087 } else if (cmd & IOC_OUT) { 1088 memset(kdata, 0, usize); 1089 } 1090 1091 retcode = func(filep, process, kdata); 1092 1093 if (cmd & IOC_OUT) 1094 if (copy_to_user((void __user *)arg, kdata, usize) != 0) 1095 retcode = -EFAULT; 1096 1097 err_i1: 1098 if (!ioctl) 1099 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 1100 task_pid_nr(current), cmd, nr); 1101 1102 if (kdata != stack_kdata) 1103 kfree(kdata); 1104 1105 if (retcode) 1106 dev_dbg(kfd_device, "ret = %d\n", retcode); 1107 1108 return retcode; 1109 } 1110 1111 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 1112 { 1113 struct kfd_process *process; 1114 1115 process = kfd_get_process(current); 1116 if (IS_ERR(process)) 1117 return PTR_ERR(process); 1118 1119 if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == 1120 KFD_MMAP_DOORBELL_MASK) { 1121 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; 1122 return kfd_doorbell_mmap(process, vma); 1123 } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == 1124 KFD_MMAP_EVENTS_MASK) { 1125 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; 1126 return kfd_event_mmap(process, vma); 1127 } else if ((vma->vm_pgoff & KFD_MMAP_RESERVED_MEM_MASK) == 1128 KFD_MMAP_RESERVED_MEM_MASK) { 1129 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_RESERVED_MEM_MASK; 1130 return kfd_reserved_mem_mmap(process, vma); 1131 } 1132 1133 return -EFAULT; 1134 } 1135