1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/device.h> 24 #include <linux/export.h> 25 #include <linux/err.h> 26 #include <linux/fs.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/uaccess.h> 30 #include <linux/compat.h> 31 #include <uapi/linux/kfd_ioctl.h> 32 #include <linux/time.h> 33 #include <linux/mm.h> 34 #include <linux/mman.h> 35 #include <asm/processor.h> 36 #include "kfd_priv.h" 37 #include "kfd_device_queue_manager.h" 38 #include "kfd_dbgmgr.h" 39 40 static long kfd_ioctl(struct file *, unsigned int, unsigned long); 41 static int kfd_open(struct inode *, struct file *); 42 static int kfd_mmap(struct file *, struct vm_area_struct *); 43 44 static const char kfd_dev_name[] = "kfd"; 45 46 static const struct file_operations kfd_fops = { 47 .owner = THIS_MODULE, 48 .unlocked_ioctl = kfd_ioctl, 49 .compat_ioctl = kfd_ioctl, 50 .open = kfd_open, 51 .mmap = kfd_mmap, 52 }; 53 54 static int kfd_char_dev_major = -1; 55 static struct class *kfd_class; 56 struct device *kfd_device; 57 58 int kfd_chardev_init(void) 59 { 60 int err = 0; 61 62 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); 63 err = kfd_char_dev_major; 64 if (err < 0) 65 goto err_register_chrdev; 66 67 kfd_class = class_create(THIS_MODULE, kfd_dev_name); 68 err = PTR_ERR(kfd_class); 69 if (IS_ERR(kfd_class)) 70 goto err_class_create; 71 72 kfd_device = device_create(kfd_class, NULL, 73 MKDEV(kfd_char_dev_major, 0), 74 NULL, kfd_dev_name); 75 err = PTR_ERR(kfd_device); 76 if (IS_ERR(kfd_device)) 77 goto err_device_create; 78 79 return 0; 80 81 err_device_create: 82 class_destroy(kfd_class); 83 err_class_create: 84 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 85 err_register_chrdev: 86 return err; 87 } 88 89 void kfd_chardev_exit(void) 90 { 91 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); 92 class_destroy(kfd_class); 93 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 94 } 95 96 struct device *kfd_chardev(void) 97 { 98 return kfd_device; 99 } 100 101 102 static int kfd_open(struct inode *inode, struct file *filep) 103 { 104 struct kfd_process *process; 105 bool is_32bit_user_mode; 106 107 if (iminor(inode) != 0) 108 return -ENODEV; 109 110 is_32bit_user_mode = is_compat_task(); 111 112 if (is_32bit_user_mode == true) { 113 dev_warn(kfd_device, 114 "Process %d (32-bit) failed to open /dev/kfd\n" 115 "32-bit processes are not supported by amdkfd\n", 116 current->pid); 117 return -EPERM; 118 } 119 120 process = kfd_create_process(current); 121 if (IS_ERR(process)) 122 return PTR_ERR(process); 123 124 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", 125 process->pasid, process->is_32bit_user_mode); 126 127 return 0; 128 } 129 130 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 131 void *data) 132 { 133 struct kfd_ioctl_get_version_args *args = data; 134 int err = 0; 135 136 args->major_version = KFD_IOCTL_MAJOR_VERSION; 137 args->minor_version = KFD_IOCTL_MINOR_VERSION; 138 139 return err; 140 } 141 142 static int set_queue_properties_from_user(struct queue_properties *q_properties, 143 struct kfd_ioctl_create_queue_args *args) 144 { 145 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 146 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 147 return -EINVAL; 148 } 149 150 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 151 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 152 return -EINVAL; 153 } 154 155 if ((args->ring_base_address) && 156 (!access_ok(VERIFY_WRITE, 157 (const void __user *) args->ring_base_address, 158 sizeof(uint64_t)))) { 159 pr_err("kfd: can't access ring base address\n"); 160 return -EFAULT; 161 } 162 163 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 164 pr_err("kfd: ring size must be a power of 2 or 0\n"); 165 return -EINVAL; 166 } 167 168 if (!access_ok(VERIFY_WRITE, 169 (const void __user *) args->read_pointer_address, 170 sizeof(uint32_t))) { 171 pr_err("kfd: can't access read pointer\n"); 172 return -EFAULT; 173 } 174 175 if (!access_ok(VERIFY_WRITE, 176 (const void __user *) args->write_pointer_address, 177 sizeof(uint32_t))) { 178 pr_err("kfd: can't access write pointer\n"); 179 return -EFAULT; 180 } 181 182 if (args->eop_buffer_address && 183 !access_ok(VERIFY_WRITE, 184 (const void __user *) args->eop_buffer_address, 185 sizeof(uint32_t))) { 186 pr_debug("kfd: can't access eop buffer"); 187 return -EFAULT; 188 } 189 190 if (args->ctx_save_restore_address && 191 !access_ok(VERIFY_WRITE, 192 (const void __user *) args->ctx_save_restore_address, 193 sizeof(uint32_t))) { 194 pr_debug("kfd: can't access ctx save restore buffer"); 195 return -EFAULT; 196 } 197 198 q_properties->is_interop = false; 199 q_properties->queue_percent = args->queue_percentage; 200 q_properties->priority = args->queue_priority; 201 q_properties->queue_address = args->ring_base_address; 202 q_properties->queue_size = args->ring_size; 203 q_properties->read_ptr = (uint32_t *) args->read_pointer_address; 204 q_properties->write_ptr = (uint32_t *) args->write_pointer_address; 205 q_properties->eop_ring_buffer_address = args->eop_buffer_address; 206 q_properties->eop_ring_buffer_size = args->eop_buffer_size; 207 q_properties->ctx_save_restore_area_address = 208 args->ctx_save_restore_address; 209 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; 210 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || 211 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 212 q_properties->type = KFD_QUEUE_TYPE_COMPUTE; 213 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) 214 q_properties->type = KFD_QUEUE_TYPE_SDMA; 215 else 216 return -ENOTSUPP; 217 218 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 219 q_properties->format = KFD_QUEUE_FORMAT_AQL; 220 else 221 q_properties->format = KFD_QUEUE_FORMAT_PM4; 222 223 pr_debug("Queue Percentage (%d, %d)\n", 224 q_properties->queue_percent, args->queue_percentage); 225 226 pr_debug("Queue Priority (%d, %d)\n", 227 q_properties->priority, args->queue_priority); 228 229 pr_debug("Queue Address (0x%llX, 0x%llX)\n", 230 q_properties->queue_address, args->ring_base_address); 231 232 pr_debug("Queue Size (0x%llX, %u)\n", 233 q_properties->queue_size, args->ring_size); 234 235 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", 236 (uint64_t) q_properties->read_ptr, 237 (uint64_t) q_properties->write_ptr); 238 239 pr_debug("Queue Format (%d)\n", q_properties->format); 240 241 pr_debug("Queue EOP (0x%llX)\n", q_properties->eop_ring_buffer_address); 242 243 pr_debug("Queue CTX save arex (0x%llX)\n", 244 q_properties->ctx_save_restore_area_address); 245 246 return 0; 247 } 248 249 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 250 void *data) 251 { 252 struct kfd_ioctl_create_queue_args *args = data; 253 struct kfd_dev *dev; 254 int err = 0; 255 unsigned int queue_id; 256 struct kfd_process_device *pdd; 257 struct queue_properties q_properties; 258 259 memset(&q_properties, 0, sizeof(struct queue_properties)); 260 261 pr_debug("kfd: creating queue ioctl\n"); 262 263 err = set_queue_properties_from_user(&q_properties, args); 264 if (err) 265 return err; 266 267 pr_debug("kfd: looking for gpu id 0x%x\n", args->gpu_id); 268 dev = kfd_device_by_id(args->gpu_id); 269 if (dev == NULL) { 270 pr_debug("kfd: gpu id 0x%x was not found\n", args->gpu_id); 271 return -EINVAL; 272 } 273 274 mutex_lock(&p->mutex); 275 276 pdd = kfd_bind_process_to_device(dev, p); 277 if (IS_ERR(pdd)) { 278 err = -ESRCH; 279 goto err_bind_process; 280 } 281 282 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", 283 p->pasid, 284 dev->id); 285 286 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 287 0, q_properties.type, &queue_id); 288 if (err != 0) 289 goto err_create_queue; 290 291 args->queue_id = queue_id; 292 293 294 /* Return gpu_id as doorbell offset for mmap usage */ 295 args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id); 296 args->doorbell_offset <<= PAGE_SHIFT; 297 298 mutex_unlock(&p->mutex); 299 300 pr_debug("kfd: queue id %d was created successfully\n", args->queue_id); 301 302 pr_debug("ring buffer address == 0x%016llX\n", 303 args->ring_base_address); 304 305 pr_debug("read ptr address == 0x%016llX\n", 306 args->read_pointer_address); 307 308 pr_debug("write ptr address == 0x%016llX\n", 309 args->write_pointer_address); 310 311 return 0; 312 313 err_create_queue: 314 err_bind_process: 315 mutex_unlock(&p->mutex); 316 return err; 317 } 318 319 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 320 void *data) 321 { 322 int retval; 323 struct kfd_ioctl_destroy_queue_args *args = data; 324 325 pr_debug("kfd: destroying queue id %d for PASID %d\n", 326 args->queue_id, 327 p->pasid); 328 329 mutex_lock(&p->mutex); 330 331 retval = pqm_destroy_queue(&p->pqm, args->queue_id); 332 333 mutex_unlock(&p->mutex); 334 return retval; 335 } 336 337 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 338 void *data) 339 { 340 int retval; 341 struct kfd_ioctl_update_queue_args *args = data; 342 struct queue_properties properties; 343 344 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 345 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 346 return -EINVAL; 347 } 348 349 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 350 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 351 return -EINVAL; 352 } 353 354 if ((args->ring_base_address) && 355 (!access_ok(VERIFY_WRITE, 356 (const void __user *) args->ring_base_address, 357 sizeof(uint64_t)))) { 358 pr_err("kfd: can't access ring base address\n"); 359 return -EFAULT; 360 } 361 362 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 363 pr_err("kfd: ring size must be a power of 2 or 0\n"); 364 return -EINVAL; 365 } 366 367 properties.queue_address = args->ring_base_address; 368 properties.queue_size = args->ring_size; 369 properties.queue_percent = args->queue_percentage; 370 properties.priority = args->queue_priority; 371 372 pr_debug("kfd: updating queue id %d for PASID %d\n", 373 args->queue_id, p->pasid); 374 375 mutex_lock(&p->mutex); 376 377 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); 378 379 mutex_unlock(&p->mutex); 380 381 return retval; 382 } 383 384 static int kfd_ioctl_set_memory_policy(struct file *filep, 385 struct kfd_process *p, void *data) 386 { 387 struct kfd_ioctl_set_memory_policy_args *args = data; 388 struct kfd_dev *dev; 389 int err = 0; 390 struct kfd_process_device *pdd; 391 enum cache_policy default_policy, alternate_policy; 392 393 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT 394 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 395 return -EINVAL; 396 } 397 398 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 399 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 400 return -EINVAL; 401 } 402 403 dev = kfd_device_by_id(args->gpu_id); 404 if (dev == NULL) 405 return -EINVAL; 406 407 mutex_lock(&p->mutex); 408 409 pdd = kfd_bind_process_to_device(dev, p); 410 if (IS_ERR(pdd)) { 411 err = -ESRCH; 412 goto out; 413 } 414 415 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 416 ? cache_policy_coherent : cache_policy_noncoherent; 417 418 alternate_policy = 419 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 420 ? cache_policy_coherent : cache_policy_noncoherent; 421 422 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, 423 &pdd->qpd, 424 default_policy, 425 alternate_policy, 426 (void __user *)args->alternate_aperture_base, 427 args->alternate_aperture_size)) 428 err = -EINVAL; 429 430 out: 431 mutex_unlock(&p->mutex); 432 433 return err; 434 } 435 436 static int kfd_ioctl_dbg_register(struct file *filep, 437 struct kfd_process *p, void *data) 438 { 439 struct kfd_ioctl_dbg_register_args *args = data; 440 struct kfd_dev *dev; 441 struct kfd_dbgmgr *dbgmgr_ptr; 442 struct kfd_process_device *pdd; 443 bool create_ok; 444 long status = 0; 445 446 dev = kfd_device_by_id(args->gpu_id); 447 if (dev == NULL) 448 return -EINVAL; 449 450 if (dev->device_info->asic_family == CHIP_CARRIZO) { 451 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); 452 return -EINVAL; 453 } 454 455 mutex_lock(kfd_get_dbgmgr_mutex()); 456 mutex_lock(&p->mutex); 457 458 /* 459 * make sure that we have pdd, if this the first queue created for 460 * this process 461 */ 462 pdd = kfd_bind_process_to_device(dev, p); 463 if (IS_ERR(pdd)) { 464 mutex_unlock(&p->mutex); 465 mutex_unlock(kfd_get_dbgmgr_mutex()); 466 return PTR_ERR(pdd); 467 } 468 469 if (dev->dbgmgr == NULL) { 470 /* In case of a legal call, we have no dbgmgr yet */ 471 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); 472 if (create_ok) { 473 status = kfd_dbgmgr_register(dbgmgr_ptr, p); 474 if (status != 0) 475 kfd_dbgmgr_destroy(dbgmgr_ptr); 476 else 477 dev->dbgmgr = dbgmgr_ptr; 478 } 479 } else { 480 pr_debug("debugger already registered\n"); 481 status = -EINVAL; 482 } 483 484 mutex_unlock(&p->mutex); 485 mutex_unlock(kfd_get_dbgmgr_mutex()); 486 487 return status; 488 } 489 490 static int kfd_ioctl_dbg_unrgesiter(struct file *filep, 491 struct kfd_process *p, void *data) 492 { 493 struct kfd_ioctl_dbg_unregister_args *args = data; 494 struct kfd_dev *dev; 495 long status; 496 497 dev = kfd_device_by_id(args->gpu_id); 498 if (dev == NULL) 499 return -EINVAL; 500 501 if (dev->device_info->asic_family == CHIP_CARRIZO) { 502 pr_debug("kfd_ioctl_dbg_unrgesiter not supported on CZ\n"); 503 return -EINVAL; 504 } 505 506 mutex_lock(kfd_get_dbgmgr_mutex()); 507 508 status = kfd_dbgmgr_unregister(dev->dbgmgr, p); 509 if (status == 0) { 510 kfd_dbgmgr_destroy(dev->dbgmgr); 511 dev->dbgmgr = NULL; 512 } 513 514 mutex_unlock(kfd_get_dbgmgr_mutex()); 515 516 return status; 517 } 518 519 /* 520 * Parse and generate variable size data structure for address watch. 521 * Total size of the buffer and # watch points is limited in order 522 * to prevent kernel abuse. (no bearing to the much smaller HW limitation 523 * which is enforced by dbgdev module) 524 * please also note that the watch address itself are not "copied from user", 525 * since it be set into the HW in user mode values. 526 * 527 */ 528 static int kfd_ioctl_dbg_address_watch(struct file *filep, 529 struct kfd_process *p, void *data) 530 { 531 struct kfd_ioctl_dbg_address_watch_args *args = data; 532 struct kfd_dev *dev; 533 struct dbg_address_watch_info aw_info; 534 unsigned char *args_buff; 535 long status; 536 void __user *cmd_from_user; 537 uint64_t watch_mask_value = 0; 538 unsigned int args_idx = 0; 539 540 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); 541 542 dev = kfd_device_by_id(args->gpu_id); 543 if (dev == NULL) 544 return -EINVAL; 545 546 if (dev->device_info->asic_family == CHIP_CARRIZO) { 547 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 548 return -EINVAL; 549 } 550 551 cmd_from_user = (void __user *) args->content_ptr; 552 553 /* Validate arguments */ 554 555 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || 556 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || 557 (cmd_from_user == NULL)) 558 return -EINVAL; 559 560 /* this is the actual buffer to work with */ 561 args_buff = memdup_user(cmd_from_user, 562 args->buf_size_in_bytes - sizeof(*args)); 563 if (IS_ERR(args_buff)) 564 return PTR_ERR(args_buff); 565 566 aw_info.process = p; 567 568 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); 569 args_idx += sizeof(aw_info.num_watch_points); 570 571 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; 572 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; 573 574 /* 575 * set watch address base pointer to point on the array base 576 * within args_buff 577 */ 578 aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; 579 580 /* skip over the addresses buffer */ 581 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; 582 583 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { 584 kfree(args_buff); 585 return -EINVAL; 586 } 587 588 watch_mask_value = (uint64_t) args_buff[args_idx]; 589 590 if (watch_mask_value > 0) { 591 /* 592 * There is an array of masks. 593 * set watch mask base pointer to point on the array base 594 * within args_buff 595 */ 596 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; 597 598 /* skip over the masks buffer */ 599 args_idx += sizeof(aw_info.watch_mask) * 600 aw_info.num_watch_points; 601 } else { 602 /* just the NULL mask, set to NULL and skip over it */ 603 aw_info.watch_mask = NULL; 604 args_idx += sizeof(aw_info.watch_mask); 605 } 606 607 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { 608 kfree(args_buff); 609 return -EINVAL; 610 } 611 612 /* Currently HSA Event is not supported for DBG */ 613 aw_info.watch_event = NULL; 614 615 mutex_lock(kfd_get_dbgmgr_mutex()); 616 617 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); 618 619 mutex_unlock(kfd_get_dbgmgr_mutex()); 620 621 kfree(args_buff); 622 623 return status; 624 } 625 626 /* Parse and generate fixed size data structure for wave control */ 627 static int kfd_ioctl_dbg_wave_control(struct file *filep, 628 struct kfd_process *p, void *data) 629 { 630 struct kfd_ioctl_dbg_wave_control_args *args = data; 631 struct kfd_dev *dev; 632 struct dbg_wave_control_info wac_info; 633 unsigned char *args_buff; 634 uint32_t computed_buff_size; 635 long status; 636 void __user *cmd_from_user; 637 unsigned int args_idx = 0; 638 639 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); 640 641 /* we use compact form, independent of the packing attribute value */ 642 computed_buff_size = sizeof(*args) + 643 sizeof(wac_info.mode) + 644 sizeof(wac_info.operand) + 645 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + 646 sizeof(wac_info.dbgWave_msg.MemoryVA) + 647 sizeof(wac_info.trapId); 648 649 dev = kfd_device_by_id(args->gpu_id); 650 if (dev == NULL) 651 return -EINVAL; 652 653 if (dev->device_info->asic_family == CHIP_CARRIZO) { 654 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 655 return -EINVAL; 656 } 657 658 /* input size must match the computed "compact" size */ 659 if (args->buf_size_in_bytes != computed_buff_size) { 660 pr_debug("size mismatch, computed : actual %u : %u\n", 661 args->buf_size_in_bytes, computed_buff_size); 662 return -EINVAL; 663 } 664 665 cmd_from_user = (void __user *) args->content_ptr; 666 667 if (cmd_from_user == NULL) 668 return -EINVAL; 669 670 /* copy the entire buffer from user */ 671 672 args_buff = memdup_user(cmd_from_user, 673 args->buf_size_in_bytes - sizeof(*args)); 674 if (IS_ERR(args_buff)) 675 return PTR_ERR(args_buff); 676 677 /* move ptr to the start of the "pay-load" area */ 678 wac_info.process = p; 679 680 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); 681 args_idx += sizeof(wac_info.operand); 682 683 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); 684 args_idx += sizeof(wac_info.mode); 685 686 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); 687 args_idx += sizeof(wac_info.trapId); 688 689 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 690 *((uint32_t *)(&args_buff[args_idx])); 691 wac_info.dbgWave_msg.MemoryVA = NULL; 692 693 mutex_lock(kfd_get_dbgmgr_mutex()); 694 695 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", 696 wac_info.process, wac_info.operand, 697 wac_info.mode, wac_info.trapId, 698 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); 699 700 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); 701 702 pr_debug("Returned status of dbg manager is %ld\n", status); 703 704 mutex_unlock(kfd_get_dbgmgr_mutex()); 705 706 kfree(args_buff); 707 708 return status; 709 } 710 711 static int kfd_ioctl_get_clock_counters(struct file *filep, 712 struct kfd_process *p, void *data) 713 { 714 struct kfd_ioctl_get_clock_counters_args *args = data; 715 struct kfd_dev *dev; 716 struct timespec64 time; 717 718 dev = kfd_device_by_id(args->gpu_id); 719 if (dev == NULL) 720 return -EINVAL; 721 722 /* Reading GPU clock counter from KGD */ 723 args->gpu_clock_counter = 724 dev->kfd2kgd->get_gpu_clock_counter(dev->kgd); 725 726 /* No access to rdtsc. Using raw monotonic time */ 727 getrawmonotonic64(&time); 728 args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time); 729 730 get_monotonic_boottime64(&time); 731 args->system_clock_counter = (uint64_t)timespec64_to_ns(&time); 732 733 /* Since the counter is in nano-seconds we use 1GHz frequency */ 734 args->system_clock_freq = 1000000000; 735 736 return 0; 737 } 738 739 740 static int kfd_ioctl_get_process_apertures(struct file *filp, 741 struct kfd_process *p, void *data) 742 { 743 struct kfd_ioctl_get_process_apertures_args *args = data; 744 struct kfd_process_device_apertures *pAperture; 745 struct kfd_process_device *pdd; 746 747 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); 748 749 args->num_of_nodes = 0; 750 751 mutex_lock(&p->mutex); 752 753 /*if the process-device list isn't empty*/ 754 if (kfd_has_process_device_data(p)) { 755 /* Run over all pdd of the process */ 756 pdd = kfd_get_first_process_device_data(p); 757 do { 758 pAperture = 759 &args->process_apertures[args->num_of_nodes]; 760 pAperture->gpu_id = pdd->dev->id; 761 pAperture->lds_base = pdd->lds_base; 762 pAperture->lds_limit = pdd->lds_limit; 763 pAperture->gpuvm_base = pdd->gpuvm_base; 764 pAperture->gpuvm_limit = pdd->gpuvm_limit; 765 pAperture->scratch_base = pdd->scratch_base; 766 pAperture->scratch_limit = pdd->scratch_limit; 767 768 dev_dbg(kfd_device, 769 "node id %u\n", args->num_of_nodes); 770 dev_dbg(kfd_device, 771 "gpu id %u\n", pdd->dev->id); 772 dev_dbg(kfd_device, 773 "lds_base %llX\n", pdd->lds_base); 774 dev_dbg(kfd_device, 775 "lds_limit %llX\n", pdd->lds_limit); 776 dev_dbg(kfd_device, 777 "gpuvm_base %llX\n", pdd->gpuvm_base); 778 dev_dbg(kfd_device, 779 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 780 dev_dbg(kfd_device, 781 "scratch_base %llX\n", pdd->scratch_base); 782 dev_dbg(kfd_device, 783 "scratch_limit %llX\n", pdd->scratch_limit); 784 785 args->num_of_nodes++; 786 } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && 787 (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 788 } 789 790 mutex_unlock(&p->mutex); 791 792 return 0; 793 } 794 795 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, 796 void *data) 797 { 798 struct kfd_ioctl_create_event_args *args = data; 799 int err; 800 801 err = kfd_event_create(filp, p, args->event_type, 802 args->auto_reset != 0, args->node_id, 803 &args->event_id, &args->event_trigger_data, 804 &args->event_page_offset, 805 &args->event_slot_index); 806 807 return err; 808 } 809 810 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, 811 void *data) 812 { 813 struct kfd_ioctl_destroy_event_args *args = data; 814 815 return kfd_event_destroy(p, args->event_id); 816 } 817 818 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, 819 void *data) 820 { 821 struct kfd_ioctl_set_event_args *args = data; 822 823 return kfd_set_event(p, args->event_id); 824 } 825 826 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, 827 void *data) 828 { 829 struct kfd_ioctl_reset_event_args *args = data; 830 831 return kfd_reset_event(p, args->event_id); 832 } 833 834 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, 835 void *data) 836 { 837 struct kfd_ioctl_wait_events_args *args = data; 838 enum kfd_event_wait_result wait_result; 839 int err; 840 841 err = kfd_wait_on_events(p, args->num_events, 842 (void __user *)args->events_ptr, 843 (args->wait_for_all != 0), 844 args->timeout, &wait_result); 845 846 args->wait_result = wait_result; 847 848 return err; 849 } 850 851 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ 852 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl} 853 854 /** Ioctl table */ 855 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { 856 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, 857 kfd_ioctl_get_version, 0), 858 859 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, 860 kfd_ioctl_create_queue, 0), 861 862 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, 863 kfd_ioctl_destroy_queue, 0), 864 865 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, 866 kfd_ioctl_set_memory_policy, 0), 867 868 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, 869 kfd_ioctl_get_clock_counters, 0), 870 871 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, 872 kfd_ioctl_get_process_apertures, 0), 873 874 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, 875 kfd_ioctl_update_queue, 0), 876 877 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, 878 kfd_ioctl_create_event, 0), 879 880 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, 881 kfd_ioctl_destroy_event, 0), 882 883 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, 884 kfd_ioctl_set_event, 0), 885 886 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, 887 kfd_ioctl_reset_event, 0), 888 889 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, 890 kfd_ioctl_wait_events, 0), 891 892 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, 893 kfd_ioctl_dbg_register, 0), 894 895 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, 896 kfd_ioctl_dbg_unrgesiter, 0), 897 898 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, 899 kfd_ioctl_dbg_address_watch, 0), 900 901 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, 902 kfd_ioctl_dbg_wave_control, 0), 903 }; 904 905 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) 906 907 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 908 { 909 struct kfd_process *process; 910 amdkfd_ioctl_t *func; 911 const struct amdkfd_ioctl_desc *ioctl = NULL; 912 unsigned int nr = _IOC_NR(cmd); 913 char stack_kdata[128]; 914 char *kdata = NULL; 915 unsigned int usize, asize; 916 int retcode = -EINVAL; 917 918 if (nr >= AMDKFD_CORE_IOCTL_COUNT) 919 goto err_i1; 920 921 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { 922 u32 amdkfd_size; 923 924 ioctl = &amdkfd_ioctls[nr]; 925 926 amdkfd_size = _IOC_SIZE(ioctl->cmd); 927 usize = asize = _IOC_SIZE(cmd); 928 if (amdkfd_size > asize) 929 asize = amdkfd_size; 930 931 cmd = ioctl->cmd; 932 } else 933 goto err_i1; 934 935 dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); 936 937 process = kfd_get_process(current); 938 if (IS_ERR(process)) { 939 dev_dbg(kfd_device, "no process\n"); 940 goto err_i1; 941 } 942 943 /* Do not trust userspace, use our own definition */ 944 func = ioctl->func; 945 946 if (unlikely(!func)) { 947 dev_dbg(kfd_device, "no function\n"); 948 retcode = -EINVAL; 949 goto err_i1; 950 } 951 952 if (cmd & (IOC_IN | IOC_OUT)) { 953 if (asize <= sizeof(stack_kdata)) { 954 kdata = stack_kdata; 955 } else { 956 kdata = kmalloc(asize, GFP_KERNEL); 957 if (!kdata) { 958 retcode = -ENOMEM; 959 goto err_i1; 960 } 961 } 962 if (asize > usize) 963 memset(kdata + usize, 0, asize - usize); 964 } 965 966 if (cmd & IOC_IN) { 967 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { 968 retcode = -EFAULT; 969 goto err_i1; 970 } 971 } else if (cmd & IOC_OUT) { 972 memset(kdata, 0, usize); 973 } 974 975 retcode = func(filep, process, kdata); 976 977 if (cmd & IOC_OUT) 978 if (copy_to_user((void __user *)arg, kdata, usize) != 0) 979 retcode = -EFAULT; 980 981 err_i1: 982 if (!ioctl) 983 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 984 task_pid_nr(current), cmd, nr); 985 986 if (kdata != stack_kdata) 987 kfree(kdata); 988 989 if (retcode) 990 dev_dbg(kfd_device, "ret = %d\n", retcode); 991 992 return retcode; 993 } 994 995 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 996 { 997 struct kfd_process *process; 998 999 process = kfd_get_process(current); 1000 if (IS_ERR(process)) 1001 return PTR_ERR(process); 1002 1003 if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == 1004 KFD_MMAP_DOORBELL_MASK) { 1005 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; 1006 return kfd_doorbell_mmap(process, vma); 1007 } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == 1008 KFD_MMAP_EVENTS_MASK) { 1009 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; 1010 return kfd_event_mmap(process, vma); 1011 } 1012 1013 return -EFAULT; 1014 } 1015