1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/slab.h> 26 #include <linux/mutex.h> 27 #include "kfd_device_queue_manager.h" 28 #include "kfd_kernel_queue.h" 29 #include "kfd_priv.h" 30 31 #define OVER_SUBSCRIPTION_PROCESS_COUNT (1 << 0) 32 #define OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT (1 << 1) 33 #define OVER_SUBSCRIPTION_GWS_QUEUE_COUNT (1 << 2) 34 #define OVER_SUBSCRIPTION_XNACK_CONFLICT (1 << 3) 35 36 static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes, 37 unsigned int buffer_size_bytes) 38 { 39 unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t); 40 41 WARN((temp * sizeof(uint32_t)) > buffer_size_bytes, 42 "Runlist IB overflow"); 43 *wptr = temp; 44 } 45 46 static void pm_calc_rlib_size(struct packet_manager *pm, 47 unsigned int *rlib_size, 48 int *over_subscription, 49 int xnack_conflict) 50 { 51 unsigned int process_count, queue_count, compute_queue_count, gws_queue_count; 52 unsigned int map_queue_size; 53 unsigned int max_proc_per_quantum = 1; 54 struct kfd_node *node = pm->dqm->dev; 55 struct device *dev = node->adev->dev; 56 57 process_count = pm->dqm->processes_count; 58 queue_count = pm->dqm->active_queue_count; 59 compute_queue_count = pm->dqm->active_cp_queue_count; 60 gws_queue_count = pm->dqm->gws_queue_count; 61 62 /* check if there is over subscription 63 * Note: the arbitration between the number of VMIDs and 64 * hws_max_conc_proc has been done in 65 * kgd2kfd_device_init(). 66 */ 67 *over_subscription = 0; 68 69 if (node->max_proc_per_quantum > 1) 70 max_proc_per_quantum = node->max_proc_per_quantum; 71 72 if (process_count > max_proc_per_quantum) 73 *over_subscription |= OVER_SUBSCRIPTION_PROCESS_COUNT; 74 if (compute_queue_count > get_cp_queues_num(pm->dqm)) 75 *over_subscription |= OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT; 76 if (gws_queue_count > 1) 77 *over_subscription |= OVER_SUBSCRIPTION_GWS_QUEUE_COUNT; 78 if (xnack_conflict && (node->adev->gmc.xnack_flags & AMDGPU_GMC_XNACK_FLAG_CHAIN)) 79 *over_subscription |= OVER_SUBSCRIPTION_XNACK_CONFLICT; 80 81 if (*over_subscription) 82 dev_dbg(dev, "Over subscribed runlist\n"); 83 84 map_queue_size = pm->pmf->map_queues_size; 85 /* calculate run list ib allocation size */ 86 *rlib_size = process_count * pm->pmf->map_process_size + 87 queue_count * map_queue_size; 88 89 /* 90 * Increase the allocation size in case we need a chained run list 91 * when over subscription 92 */ 93 if (*over_subscription) 94 *rlib_size += pm->pmf->runlist_size; 95 96 dev_dbg(dev, "runlist ib size %d\n", *rlib_size); 97 } 98 99 static int pm_allocate_runlist_ib(struct packet_manager *pm, 100 unsigned int **rl_buffer, 101 uint64_t *rl_gpu_buffer, 102 unsigned int *rl_buffer_size, 103 int *is_over_subscription, 104 int xnack_conflict) 105 { 106 struct kfd_node *node = pm->dqm->dev; 107 struct device *dev = node->adev->dev; 108 int retval; 109 110 if (WARN_ON(pm->allocated)) 111 return -EINVAL; 112 113 pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription, 114 xnack_conflict); 115 116 mutex_lock(&pm->lock); 117 118 retval = kfd_gtt_sa_allocate(node, *rl_buffer_size, &pm->ib_buffer_obj); 119 120 if (retval) { 121 dev_err(dev, "Failed to allocate runlist IB\n"); 122 goto out; 123 } 124 125 *(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr; 126 *rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr; 127 128 memset(*rl_buffer, 0, *rl_buffer_size); 129 pm->allocated = true; 130 131 out: 132 mutex_unlock(&pm->lock); 133 return retval; 134 } 135 136 static int pm_create_runlist_ib(struct packet_manager *pm, 137 struct list_head *queues, 138 uint64_t *rl_gpu_addr, 139 size_t *rl_size_bytes) 140 { 141 unsigned int alloc_size_bytes; 142 unsigned int *rl_buffer, rl_wptr, i; 143 struct kfd_node *node = pm->dqm->dev; 144 struct device *dev = node->adev->dev; 145 int retval, processes_mapped; 146 struct device_process_node *cur; 147 struct qcm_process_device *qpd; 148 struct queue *q; 149 struct kernel_queue *kq; 150 int is_over_subscription; 151 int xnack_enabled = -1; 152 bool xnack_conflict = 0; 153 154 rl_wptr = retval = processes_mapped = 0; 155 156 /* Check if processes set different xnack modes */ 157 list_for_each_entry(cur, queues, list) { 158 qpd = cur->qpd; 159 if (xnack_enabled < 0) 160 /* First process */ 161 xnack_enabled = qpd->pqm->process->xnack_enabled; 162 else if (qpd->pqm->process->xnack_enabled != xnack_enabled) { 163 /* Found a process with a different xnack mode */ 164 xnack_conflict = 1; 165 break; 166 } 167 } 168 169 retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr, 170 &alloc_size_bytes, &is_over_subscription, 171 xnack_conflict); 172 if (retval) 173 return retval; 174 175 *rl_size_bytes = alloc_size_bytes; 176 pm->ib_size_bytes = alloc_size_bytes; 177 178 dev_dbg(dev, "Building runlist ib process count: %d queues count %d\n", 179 pm->dqm->processes_count, pm->dqm->active_queue_count); 180 181 build_runlist_ib: 182 /* build the run list ib packet */ 183 list_for_each_entry(cur, queues, list) { 184 qpd = cur->qpd; 185 /* group processes with the same xnack mode together */ 186 if (qpd->pqm->process->xnack_enabled != xnack_enabled) 187 continue; 188 /* build map process packet */ 189 if (processes_mapped >= pm->dqm->processes_count) { 190 dev_dbg(dev, "Not enough space left in runlist IB\n"); 191 pm_release_ib(pm); 192 return -ENOMEM; 193 } 194 195 retval = pm->pmf->map_process(pm, &rl_buffer[rl_wptr], qpd); 196 if (retval) 197 return retval; 198 199 processes_mapped++; 200 inc_wptr(&rl_wptr, pm->pmf->map_process_size, 201 alloc_size_bytes); 202 203 list_for_each_entry(kq, &qpd->priv_queue_list, list) { 204 if (!kq->queue->properties.is_active) 205 continue; 206 207 dev_dbg(dev, 208 "static_queue, mapping kernel q %d, is debug status %d\n", 209 kq->queue->queue, qpd->is_debug); 210 211 retval = pm->pmf->map_queues(pm, 212 &rl_buffer[rl_wptr], 213 kq->queue, 214 qpd->is_debug); 215 if (retval) 216 return retval; 217 218 inc_wptr(&rl_wptr, 219 pm->pmf->map_queues_size, 220 alloc_size_bytes); 221 } 222 223 list_for_each_entry(q, &qpd->queues_list, list) { 224 if (!q->properties.is_active) 225 continue; 226 227 dev_dbg(dev, 228 "static_queue, mapping user queue %d, is debug status %d\n", 229 q->queue, qpd->is_debug); 230 231 retval = pm->pmf->map_queues(pm, 232 &rl_buffer[rl_wptr], 233 q, 234 qpd->is_debug); 235 236 if (retval) 237 return retval; 238 239 inc_wptr(&rl_wptr, 240 pm->pmf->map_queues_size, 241 alloc_size_bytes); 242 } 243 } 244 if (xnack_conflict) { 245 /* pick up processes with the other xnack mode */ 246 xnack_enabled = !xnack_enabled; 247 xnack_conflict = 0; 248 goto build_runlist_ib; 249 } 250 251 dev_dbg(dev, "Finished map process and queues to runlist\n"); 252 253 if (is_over_subscription) { 254 if (!pm->is_over_subscription) 255 dev_warn(dev, "Runlist is getting oversubscribed due to%s%s%s%s. Expect reduced ROCm performance.\n", 256 is_over_subscription & OVER_SUBSCRIPTION_PROCESS_COUNT ? 257 " too many processes" : "", 258 is_over_subscription & OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT ? 259 " too many queues" : "", 260 is_over_subscription & OVER_SUBSCRIPTION_GWS_QUEUE_COUNT ? 261 " multiple processes using cooperative launch" : "", 262 is_over_subscription & OVER_SUBSCRIPTION_XNACK_CONFLICT ? 263 " xnack on/off processes mixed on gfx9" : ""); 264 265 retval = pm->pmf->runlist(pm, &rl_buffer[rl_wptr], 266 *rl_gpu_addr, 267 alloc_size_bytes / sizeof(uint32_t), 268 true); 269 } 270 pm->is_over_subscription = !!is_over_subscription; 271 272 pr_debug("Runlist dump:"); 273 for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i += 8) { 274 char buf[128]; 275 int j, len = 0; 276 277 /* Dump 8 entries per line with an index for each line */ 278 len += scnprintf(buf + len, sizeof(buf) - len, "%4u:", i); 279 280 for (j = 0; j < 8 && (i + j) < alloc_size_bytes / sizeof(uint32_t); j++) 281 len += scnprintf(buf + len, sizeof(buf) - len, " 0x%08x", rl_buffer[i + j]); 282 283 pr_debug("%s\n", buf); 284 } 285 286 return retval; 287 } 288 289 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm) 290 { 291 switch (dqm->dev->adev->asic_type) { 292 case CHIP_KAVERI: 293 case CHIP_HAWAII: 294 /* PM4 packet structures on CIK are the same as on VI */ 295 case CHIP_CARRIZO: 296 case CHIP_TONGA: 297 case CHIP_FIJI: 298 case CHIP_POLARIS10: 299 case CHIP_POLARIS11: 300 case CHIP_POLARIS12: 301 case CHIP_VEGAM: 302 pm->pmf = &kfd_vi_pm_funcs; 303 break; 304 default: 305 if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 2) || 306 KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 3) || 307 KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 4) || 308 KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 5, 0)) 309 pm->pmf = &kfd_aldebaran_pm_funcs; 310 else if (KFD_GC_VERSION(dqm->dev) >= IP_VERSION(9, 0, 1)) 311 pm->pmf = &kfd_v9_pm_funcs; 312 else { 313 WARN(1, "Unexpected ASIC family %u", 314 dqm->dev->adev->asic_type); 315 return -EINVAL; 316 } 317 } 318 319 pm->dqm = dqm; 320 mutex_init(&pm->lock); 321 pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ); 322 if (!pm->priv_queue) { 323 mutex_destroy(&pm->lock); 324 return -ENOMEM; 325 } 326 pm->allocated = false; 327 328 return 0; 329 } 330 331 void pm_uninit(struct packet_manager *pm) 332 { 333 mutex_destroy(&pm->lock); 334 kernel_queue_uninit(pm->priv_queue); 335 pm->priv_queue = NULL; 336 } 337 338 int pm_send_set_resources(struct packet_manager *pm, 339 struct scheduling_resources *res) 340 { 341 struct kfd_node *node = pm->dqm->dev; 342 struct device *dev = node->adev->dev; 343 uint32_t *buffer, size; 344 int retval = 0; 345 346 size = pm->pmf->set_resources_size; 347 mutex_lock(&pm->lock); 348 kq_acquire_packet_buffer(pm->priv_queue, 349 size / sizeof(uint32_t), 350 (unsigned int **)&buffer); 351 if (!buffer) { 352 dev_err(dev, "Failed to allocate buffer on kernel queue\n"); 353 retval = -ENOMEM; 354 goto out; 355 } 356 357 retval = pm->pmf->set_resources(pm, buffer, res); 358 if (!retval) 359 retval = kq_submit_packet(pm->priv_queue); 360 else 361 kq_rollback_packet(pm->priv_queue); 362 363 out: 364 mutex_unlock(&pm->lock); 365 366 return retval; 367 } 368 369 int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues) 370 { 371 uint64_t rl_gpu_ib_addr; 372 uint32_t *rl_buffer; 373 size_t rl_ib_size, packet_size_dwords; 374 int retval; 375 376 retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr, 377 &rl_ib_size); 378 if (retval) 379 goto fail_create_runlist_ib; 380 381 pr_debug("runlist IB address: 0x%llX\n", rl_gpu_ib_addr); 382 383 packet_size_dwords = pm->pmf->runlist_size / sizeof(uint32_t); 384 mutex_lock(&pm->lock); 385 386 retval = kq_acquire_packet_buffer(pm->priv_queue, 387 packet_size_dwords, &rl_buffer); 388 if (retval) 389 goto fail_acquire_packet_buffer; 390 391 retval = pm->pmf->runlist(pm, rl_buffer, rl_gpu_ib_addr, 392 rl_ib_size / sizeof(uint32_t), false); 393 if (retval) 394 goto fail_create_runlist; 395 396 retval = kq_submit_packet(pm->priv_queue); 397 398 mutex_unlock(&pm->lock); 399 400 return retval; 401 402 fail_create_runlist: 403 kq_rollback_packet(pm->priv_queue); 404 fail_acquire_packet_buffer: 405 mutex_unlock(&pm->lock); 406 fail_create_runlist_ib: 407 pm_release_ib(pm); 408 return retval; 409 } 410 411 int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, 412 uint64_t fence_value) 413 { 414 struct kfd_node *node = pm->dqm->dev; 415 struct device *dev = node->adev->dev; 416 uint32_t *buffer, size; 417 int retval = 0; 418 419 if (WARN_ON(!fence_address)) 420 return -EFAULT; 421 422 size = pm->pmf->query_status_size; 423 mutex_lock(&pm->lock); 424 kq_acquire_packet_buffer(pm->priv_queue, 425 size / sizeof(uint32_t), (unsigned int **)&buffer); 426 if (!buffer) { 427 dev_err(dev, "Failed to allocate buffer on kernel queue\n"); 428 retval = -ENOMEM; 429 goto out; 430 } 431 432 retval = pm->pmf->query_status(pm, buffer, fence_address, fence_value); 433 if (!retval) 434 retval = kq_submit_packet(pm->priv_queue); 435 else 436 kq_rollback_packet(pm->priv_queue); 437 438 out: 439 mutex_unlock(&pm->lock); 440 return retval; 441 } 442 443 /* pm_config_dequeue_wait_counts: Configure dequeue timer Wait Counts 444 * by writing to CP_IQ_WAIT_TIME2 registers. 445 * 446 * @cmd: See emum kfd_config_dequeue_wait_counts_cmd definition 447 * @value: Depends on the cmd. This parameter is unused for 448 * KFD_DEQUEUE_WAIT_INIT and KFD_DEQUEUE_WAIT_RESET. For 449 * KFD_DEQUEUE_WAIT_SET_SCH_WAVE it holds value to be set 450 * 451 */ 452 int pm_config_dequeue_wait_counts(struct packet_manager *pm, 453 enum kfd_config_dequeue_wait_counts_cmd cmd, 454 uint32_t value) 455 { 456 struct kfd_node *node = pm->dqm->dev; 457 struct device *dev = node->adev->dev; 458 int retval = 0; 459 uint32_t *buffer, size; 460 461 if (!pm->pmf->config_dequeue_wait_counts || 462 !pm->pmf->config_dequeue_wait_counts_size) 463 return 0; 464 465 if (cmd == KFD_DEQUEUE_WAIT_INIT && (KFD_GC_VERSION(pm->dqm->dev) < IP_VERSION(9, 4, 1) || 466 KFD_GC_VERSION(pm->dqm->dev) >= IP_VERSION(10, 0, 0))) 467 return 0; 468 469 size = pm->pmf->config_dequeue_wait_counts_size; 470 471 mutex_lock(&pm->lock); 472 473 if (size) { 474 kq_acquire_packet_buffer(pm->priv_queue, 475 size / sizeof(uint32_t), 476 (unsigned int **)&buffer); 477 478 if (!buffer) { 479 dev_err(dev, 480 "Failed to allocate buffer on kernel queue\n"); 481 retval = -ENOMEM; 482 goto out; 483 } 484 485 retval = pm->pmf->config_dequeue_wait_counts(pm, buffer, 486 cmd, value); 487 if (!retval) { 488 retval = kq_submit_packet(pm->priv_queue); 489 490 /* If default value is modified, cache that in dqm->wait_times */ 491 if (!retval && cmd == KFD_DEQUEUE_WAIT_INIT) 492 update_dqm_wait_times(pm->dqm); 493 } else { 494 kq_rollback_packet(pm->priv_queue); 495 } 496 } 497 out: 498 mutex_unlock(&pm->lock); 499 return retval; 500 } 501 502 int pm_send_unmap_queue(struct packet_manager *pm, 503 enum kfd_unmap_queues_filter filter, 504 uint32_t filter_param, bool reset) 505 { 506 struct kfd_node *node = pm->dqm->dev; 507 struct device *dev = node->adev->dev; 508 uint32_t *buffer, size; 509 int retval = 0; 510 511 size = pm->pmf->unmap_queues_size; 512 mutex_lock(&pm->lock); 513 kq_acquire_packet_buffer(pm->priv_queue, 514 size / sizeof(uint32_t), (unsigned int **)&buffer); 515 if (!buffer) { 516 dev_err(dev, "Failed to allocate buffer on kernel queue\n"); 517 retval = -ENOMEM; 518 goto out; 519 } 520 521 retval = pm->pmf->unmap_queues(pm, buffer, filter, filter_param, reset); 522 if (!retval) 523 retval = kq_submit_packet(pm->priv_queue); 524 else 525 kq_rollback_packet(pm->priv_queue); 526 527 out: 528 mutex_unlock(&pm->lock); 529 return retval; 530 } 531 532 void pm_release_ib(struct packet_manager *pm) 533 { 534 mutex_lock(&pm->lock); 535 if (pm->allocated) { 536 kfd_gtt_sa_free(pm->dqm->dev, pm->ib_buffer_obj); 537 pm->allocated = false; 538 } 539 mutex_unlock(&pm->lock); 540 } 541 542 #if defined(CONFIG_DEBUG_FS) 543 544 int pm_debugfs_runlist(struct seq_file *m, void *data) 545 { 546 struct packet_manager *pm = data; 547 548 mutex_lock(&pm->lock); 549 550 if (!pm->allocated) { 551 seq_puts(m, " No active runlist\n"); 552 goto out; 553 } 554 555 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, 556 pm->ib_buffer_obj->cpu_ptr, pm->ib_size_bytes, false); 557 558 out: 559 mutex_unlock(&pm->lock); 560 return 0; 561 } 562 563 int pm_debugfs_hang_hws(struct packet_manager *pm) 564 { 565 struct kfd_node *node = pm->dqm->dev; 566 struct device *dev = node->adev->dev; 567 uint32_t *buffer, size; 568 int r = 0; 569 570 if (!pm->priv_queue) 571 return -EAGAIN; 572 573 size = pm->pmf->query_status_size; 574 mutex_lock(&pm->lock); 575 kq_acquire_packet_buffer(pm->priv_queue, 576 size / sizeof(uint32_t), (unsigned int **)&buffer); 577 if (!buffer) { 578 dev_err(dev, "Failed to allocate buffer on kernel queue\n"); 579 r = -ENOMEM; 580 goto out; 581 } 582 memset(buffer, 0x55, size); 583 kq_submit_packet(pm->priv_queue); 584 585 dev_info(dev, "Submitting %x %x %x %x %x %x %x to HIQ to hang the HWS.", 586 buffer[0], buffer[1], buffer[2], buffer[3], buffer[4], 587 buffer[5], buffer[6]); 588 out: 589 mutex_unlock(&pm->lock); 590 return r; 591 } 592 593 594 #endif 595