1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2015 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: AMD 24 * 25 */ 26 27 #include "dm_services_types.h" 28 #include "dc.h" 29 #include "dc/dc_dmub_srv.h" 30 #include "dc/dc_stat.h" 31 32 #include "amdgpu.h" 33 #include "amdgpu_display.h" 34 #include "amdgpu_dm.h" 35 #include "amdgpu_dm_irq.h" 36 #include "dm_helpers.h" 37 #include "amdgpu_dm_crtc.h" 38 #include "amdgpu_dm_hdcp.h" 39 #include "amdgpu_dm_mst_types.h" 40 #include "amdgpu_dm_dmub.h" 41 #include "amdgpu_dm_trace.h" 42 #include "link/protocols/link_dpcd.h" 43 #include "link_service_types.h" 44 #include "ivsrcid/ivsrcid_vislands30.h" 45 #include "ivsrcid/dcn/irqsrcs_dcn_1_0.h" 46 #include "modules/inc/mod_freesync.h" 47 #include <drm/drm_vblank.h> 48 49 /** 50 * DOC: overview 51 * 52 * DM provides another layer of IRQ management on top of what the base driver 53 * already provides. This is something that could be cleaned up, and is a 54 * future TODO item. 55 * 56 * The base driver provides IRQ source registration with DRM, handler 57 * registration into the base driver's IRQ table, and a handler callback 58 * amdgpu_irq_handler(), with which DRM calls on interrupts. This generic 59 * handler looks up the IRQ table, and calls the respective 60 * &amdgpu_irq_src_funcs.process hookups. 61 * 62 * What DM provides on top are two IRQ tables specifically for top-half and 63 * bottom-half IRQ handling, with the bottom-half implementing workqueues: 64 * 65 * - &amdgpu_display_manager.irq_handler_list_high_tab 66 * - &amdgpu_display_manager.irq_handler_list_low_tab 67 * 68 * They override the base driver's IRQ table, and the effect can be seen 69 * in the hooks that DM provides for &amdgpu_irq_src_funcs.process. They 70 * are all set to the DM generic handler amdgpu_dm_irq_handler(), which looks up 71 * DM's IRQ tables. However, in order for base driver to recognize this hook, DM 72 * still needs to register the IRQ with the base driver. See 73 * amdgpu_dm_dce110_register_irq_handlers() and 74 * amdgpu_dm_dcn10_register_irq_handlers(). 75 * 76 * To expose DC's hardware interrupt toggle to the base driver, DM implements 77 * &amdgpu_irq_src_funcs.set hooks. Base driver calls it through 78 * amdgpu_irq_update() to enable or disable the interrupt. 79 */ 80 81 /****************************************************************************** 82 * Private declarations. 83 *****************************************************************************/ 84 85 /** 86 * struct amdgpu_dm_irq_handler_data - Data for DM interrupt handlers. 87 * 88 * @list: Linked list entry referencing the next/previous handler 89 * @handler: Handler function 90 * @handler_arg: Argument passed to the handler when triggered 91 * @dm: DM which this handler belongs to 92 * @irq_source: DC interrupt source that this handler is registered for 93 * @work: work struct 94 */ 95 struct amdgpu_dm_irq_handler_data { 96 struct list_head list; 97 interrupt_handler handler; 98 void *handler_arg; 99 100 struct amdgpu_display_manager *dm; 101 /* DAL irq source which registered for this interrupt. */ 102 enum dc_irq_source irq_source; 103 struct work_struct work; 104 }; 105 106 #define DM_IRQ_TABLE_LOCK(adev, flags) \ 107 spin_lock_irqsave(&adev->dm.irq_handler_list_table_lock, flags) 108 109 #define DM_IRQ_TABLE_UNLOCK(adev, flags) \ 110 spin_unlock_irqrestore(&adev->dm.irq_handler_list_table_lock, flags) 111 112 /****************************************************************************** 113 * Private functions. 114 *****************************************************************************/ 115 116 static void init_handler_common_data(struct amdgpu_dm_irq_handler_data *hcd, 117 void (*ih)(void *), 118 void *args, 119 struct amdgpu_display_manager *dm) 120 { 121 hcd->handler = ih; 122 hcd->handler_arg = args; 123 hcd->dm = dm; 124 } 125 126 /** 127 * dm_irq_work_func() - Handle an IRQ outside of the interrupt handler proper. 128 * 129 * @work: work struct 130 */ 131 static void dm_irq_work_func(struct work_struct *work) 132 { 133 struct amdgpu_dm_irq_handler_data *handler_data = 134 container_of(work, struct amdgpu_dm_irq_handler_data, work); 135 136 handler_data->handler(handler_data->handler_arg); 137 138 /* Call a DAL subcomponent which registered for interrupt notification 139 * at INTERRUPT_LOW_IRQ_CONTEXT. 140 * (The most common use is HPD interrupt) 141 */ 142 } 143 144 /* 145 * Remove a handler and return a pointer to handler list from which the 146 * handler was removed. 147 */ 148 static struct list_head *remove_irq_handler(struct amdgpu_device *adev, 149 void *ih, 150 const struct dc_interrupt_params *int_params) 151 { 152 struct list_head *hnd_list; 153 struct list_head *entry, *tmp; 154 struct amdgpu_dm_irq_handler_data *handler; 155 unsigned long irq_table_flags; 156 bool handler_removed = false; 157 enum dc_irq_source irq_source; 158 159 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 160 161 irq_source = int_params->irq_source; 162 163 switch (int_params->int_context) { 164 case INTERRUPT_HIGH_IRQ_CONTEXT: 165 hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source]; 166 break; 167 case INTERRUPT_LOW_IRQ_CONTEXT: 168 default: 169 hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source]; 170 break; 171 } 172 173 list_for_each_safe(entry, tmp, hnd_list) { 174 175 handler = list_entry(entry, struct amdgpu_dm_irq_handler_data, 176 list); 177 178 if (handler == NULL) 179 continue; 180 181 if (ih == handler->handler) { 182 /* Found our handler. Remove it from the list. */ 183 list_del(&handler->list); 184 handler_removed = true; 185 break; 186 } 187 } 188 189 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 190 191 if (!handler_removed) { 192 /* Not necessarily an error - caller may not 193 * know the context. 194 */ 195 return NULL; 196 } 197 198 if (int_params->int_context == INTERRUPT_LOW_IRQ_CONTEXT) 199 cancel_work_sync(&handler->work); 200 201 kfree(handler); 202 203 DRM_DEBUG_KMS( 204 "DM_IRQ: removed irq handler: %p for: dal_src=%d, irq context=%d\n", 205 ih, int_params->irq_source, int_params->int_context); 206 207 return hnd_list; 208 } 209 210 static bool 211 validate_irq_registration_params(struct dc_interrupt_params *int_params, 212 void (*ih)(void *)) 213 { 214 if (NULL == int_params || NULL == ih) { 215 DRM_ERROR("DM_IRQ: invalid input!\n"); 216 return false; 217 } 218 219 if (int_params->int_context >= INTERRUPT_CONTEXT_NUMBER) { 220 DRM_ERROR("DM_IRQ: invalid context: %d!\n", 221 int_params->int_context); 222 return false; 223 } 224 225 if (!DAL_VALID_IRQ_SRC_NUM(int_params->irq_source)) { 226 DRM_ERROR("DM_IRQ: invalid irq_source: %d!\n", 227 int_params->irq_source); 228 return false; 229 } 230 231 return true; 232 } 233 234 static bool validate_irq_unregistration_params(enum dc_irq_source irq_source, 235 irq_handler_idx handler_idx) 236 { 237 if (handler_idx == DAL_INVALID_IRQ_HANDLER_IDX) { 238 DRM_ERROR("DM_IRQ: invalid handler_idx==NULL!\n"); 239 return false; 240 } 241 242 if (!DAL_VALID_IRQ_SRC_NUM(irq_source)) { 243 DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", irq_source); 244 return false; 245 } 246 247 return true; 248 } 249 /****************************************************************************** 250 * Public functions. 251 * 252 * Note: caller is responsible for input validation. 253 *****************************************************************************/ 254 255 /** 256 * amdgpu_dm_irq_register_interrupt() - Register a handler within DM. 257 * @adev: The base driver device containing the DM device. 258 * @int_params: Interrupt parameters containing the source, and handler context 259 * @ih: Function pointer to the interrupt handler to register 260 * @handler_args: Arguments passed to the handler when the interrupt occurs 261 * 262 * Register an interrupt handler for the given IRQ source, under the given 263 * context. The context can either be high or low. High context handlers are 264 * executed directly within ISR context, while low context is executed within a 265 * workqueue, thereby allowing operations that sleep. 266 * 267 * Registered handlers are called in a FIFO manner, i.e. the most recently 268 * registered handler will be called first. 269 * 270 * Return: Handler data &struct amdgpu_dm_irq_handler_data containing the IRQ 271 * source, handler function, and args 272 */ 273 void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev, 274 struct dc_interrupt_params *int_params, 275 void (*ih)(void *), 276 void *handler_args) 277 { 278 struct list_head *hnd_list; 279 struct amdgpu_dm_irq_handler_data *handler_data; 280 unsigned long irq_table_flags; 281 enum dc_irq_source irq_source; 282 283 if (!validate_irq_registration_params(int_params, ih)) 284 return DAL_INVALID_IRQ_HANDLER_IDX; 285 286 handler_data = kzalloc_obj(*handler_data); 287 if (!handler_data) { 288 DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n"); 289 return DAL_INVALID_IRQ_HANDLER_IDX; 290 } 291 292 init_handler_common_data(handler_data, ih, handler_args, &adev->dm); 293 294 irq_source = int_params->irq_source; 295 296 handler_data->irq_source = irq_source; 297 298 /* Lock the list, add the handler. */ 299 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 300 301 switch (int_params->int_context) { 302 case INTERRUPT_HIGH_IRQ_CONTEXT: 303 hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source]; 304 break; 305 case INTERRUPT_LOW_IRQ_CONTEXT: 306 default: 307 hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source]; 308 INIT_WORK(&handler_data->work, dm_irq_work_func); 309 break; 310 } 311 312 list_add_tail(&handler_data->list, hnd_list); 313 314 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 315 316 /* This pointer will be stored by code which requested interrupt 317 * registration. 318 * The same pointer will be needed in order to unregister the 319 * interrupt. 320 */ 321 322 DRM_DEBUG_KMS( 323 "DM_IRQ: added irq handler: %p for: dal_src=%d, irq context=%d\n", 324 handler_data, 325 irq_source, 326 int_params->int_context); 327 328 return handler_data; 329 } 330 EXPORT_IF_KUNIT(amdgpu_dm_irq_register_interrupt); 331 332 /** 333 * amdgpu_dm_irq_unregister_interrupt() - Remove a handler from the DM IRQ table 334 * @adev: The base driver device containing the DM device 335 * @irq_source: IRQ source to remove the given handler from 336 * @ih: Function pointer to the interrupt handler to unregister 337 * 338 * Go through both low and high context IRQ tables, and find the given handler 339 * for the given irq source. If found, remove it. Otherwise, do nothing. 340 */ 341 void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev, 342 enum dc_irq_source irq_source, 343 void *ih) 344 { 345 struct list_head *handler_list; 346 struct dc_interrupt_params int_params; 347 int i; 348 349 if (!validate_irq_unregistration_params(irq_source, ih)) 350 return; 351 352 memset(&int_params, 0, sizeof(int_params)); 353 354 int_params.irq_source = irq_source; 355 356 for (i = 0; i < INTERRUPT_CONTEXT_NUMBER; i++) { 357 358 int_params.int_context = i; 359 360 handler_list = remove_irq_handler(adev, ih, &int_params); 361 362 if (handler_list != NULL) 363 break; 364 } 365 366 if (handler_list == NULL) { 367 /* If we got here, it means we searched all irq contexts 368 * for this irq source, but the handler was not found. 369 */ 370 DRM_ERROR( 371 "DM_IRQ: failed to find irq handler:%p for irq_source:%d!\n", 372 ih, irq_source); 373 } 374 } 375 EXPORT_IF_KUNIT(amdgpu_dm_irq_unregister_interrupt); 376 377 /** 378 * amdgpu_dm_irq_init() - Initialize DM IRQ management 379 * @adev: The base driver device containing the DM device 380 * 381 * Initialize DM's high and low context IRQ tables. 382 * 383 * The N by M table contains N IRQ sources, with M 384 * &struct amdgpu_dm_irq_handler_data hooked together in a linked list. The 385 * list_heads are initialized here. When an interrupt n is triggered, all m 386 * handlers are called in sequence, FIFO according to registration order. 387 * 388 * The low context table requires special steps to initialize, since handlers 389 * will be deferred to a workqueue. See &struct irq_list_head. 390 */ 391 int amdgpu_dm_irq_init(struct amdgpu_device *adev) 392 { 393 int src; 394 struct list_head *lh; 395 396 DRM_DEBUG_KMS("DM_IRQ\n"); 397 398 spin_lock_init(&adev->dm.irq_handler_list_table_lock); 399 spin_lock_init(&adev->dm.irq_reg_lock); 400 401 adev->dm.irq_wq = alloc_workqueue("amdgpu_dm_irq", 402 WQ_UNBOUND | WQ_HIGHPRI, 0); 403 404 if (!adev->dm.irq_wq) 405 return -ENOMEM; 406 407 adev->dm.vmin_vmax_wq = alloc_workqueue("amdgpu_dm_vmin_vmax", 408 WQ_UNBOUND, 0); 409 410 if (!adev->dm.vmin_vmax_wq) { 411 destroy_workqueue(adev->dm.irq_wq); 412 adev->dm.irq_wq = NULL; 413 return -ENOMEM; 414 } 415 416 for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { 417 /* low context handler list init */ 418 lh = &adev->dm.irq_handler_list_low_tab[src]; 419 INIT_LIST_HEAD(lh); 420 /* high context handler init */ 421 INIT_LIST_HEAD(&adev->dm.irq_handler_list_high_tab[src]); 422 } 423 424 return 0; 425 } 426 EXPORT_IF_KUNIT(amdgpu_dm_irq_init); 427 428 /** 429 * amdgpu_dm_irq_fini() - Tear down DM IRQ management 430 * @adev: The base driver device containing the DM device 431 * 432 * Removes all handlers from the IRQ tables under the spinlock, cancels 433 * pending work items, and deallocates all handler data. 434 */ 435 void amdgpu_dm_irq_fini(struct amdgpu_device *adev) 436 { 437 int src; 438 LIST_HEAD(low_handlers); 439 LIST_HEAD(high_handlers); 440 struct list_head *entry, *tmp; 441 struct amdgpu_dm_irq_handler_data *handler; 442 unsigned long irq_table_flags; 443 444 DRM_DEBUG_KMS("DM_IRQ: releasing resources.\n"); 445 446 for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { 447 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 448 449 /* 450 * Move all handlers from the low and high context tables to 451 * temporary lists under the lock. This prevents the ISR from 452 * finding them while we process them outside the lock. 453 */ 454 list_splice_init(&adev->dm.irq_handler_list_low_tab[src], 455 &low_handlers); 456 list_splice_init(&adev->dm.irq_handler_list_high_tab[src], 457 &high_handlers); 458 459 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 460 461 /* 462 * Cancel all pending work for the low-context handlers 463 * outside the lock. cancel_work_sync() may sleep and waits 464 * until any running work completes, preventing UAF. 465 */ 466 list_for_each_safe(entry, tmp, &low_handlers) { 467 handler = list_entry(entry, 468 struct amdgpu_dm_irq_handler_data, 469 list); 470 cancel_work_sync(&handler->work); 471 } 472 473 /* 474 * High-context handlers are executed synchronously within ISR 475 * context (see amdgpu_dm_irq_immediate_work()) and have no 476 * work_struct, so there is no pending work to cancel here. 477 * They will be freed along with low_handlers after the loop. 478 */ 479 } 480 481 /* Deallocate all handlers. */ 482 list_for_each_safe(entry, tmp, &low_handlers) { 483 handler = list_entry(entry, 484 struct amdgpu_dm_irq_handler_data, 485 list); 486 list_del(&handler->list); 487 kfree(handler); 488 } 489 490 list_for_each_safe(entry, tmp, &high_handlers) { 491 handler = list_entry(entry, 492 struct amdgpu_dm_irq_handler_data, 493 list); 494 list_del(&handler->list); 495 kfree(handler); 496 } 497 498 if (adev->dm.vmin_vmax_wq) { 499 destroy_workqueue(adev->dm.vmin_vmax_wq); 500 adev->dm.vmin_vmax_wq = NULL; 501 } 502 503 if (adev->dm.irq_wq) { 504 destroy_workqueue(adev->dm.irq_wq); 505 adev->dm.irq_wq = NULL; 506 } 507 } 508 EXPORT_IF_KUNIT(amdgpu_dm_irq_fini); 509 510 void amdgpu_dm_irq_suspend(struct amdgpu_device *adev) 511 { 512 struct drm_device *dev = adev_to_drm(adev); 513 int src; 514 struct list_head *hnd_list_l; 515 unsigned long irq_table_flags; 516 struct list_head *entry, *tmp; 517 struct amdgpu_dm_irq_handler_data *handler; 518 519 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 520 521 DRM_DEBUG_KMS("DM_IRQ: suspend\n"); 522 523 /** 524 * Disable HW interrupt for HPD and HPDRX only since FLIP and VBLANK 525 * will be disabled from manage_dm_interrupts on disable CRTC. 526 * 527 * Disable the HW interrupt first, then flush any pending work. Since 528 * the HW interrupt is disabled under the lock, no new IRQ can be 529 * generated after the disable completes. Any work already queued by an 530 * in-flight ISR will be flushed below. 531 */ 532 for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6RX; src++) { 533 hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; 534 amdgpu_dm_irq_set(adev, src, false); 535 536 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 537 538 if (!list_empty(hnd_list_l)) { 539 list_for_each_safe(entry, tmp, hnd_list_l) { 540 handler = list_entry( 541 entry, 542 struct amdgpu_dm_irq_handler_data, 543 list); 544 flush_work(&handler->work); 545 } 546 } 547 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 548 } 549 550 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 551 552 if (dev->mode_config.poll_enabled) 553 drm_kms_helper_poll_disable(dev); 554 } 555 EXPORT_IF_KUNIT(amdgpu_dm_irq_suspend); 556 557 void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev) 558 { 559 int src; 560 struct list_head *hnd_list_h, *hnd_list_l; 561 unsigned long irq_table_flags; 562 563 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 564 565 drm_dbg(adev_to_drm(adev), "DM_IRQ: early resume\n"); 566 567 /* re-enable short pulse interrupts HW interrupt */ 568 for (src = DC_IRQ_SOURCE_HPD1RX; src <= DC_IRQ_SOURCE_HPD6RX; src++) { 569 hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; 570 hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; 571 if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) 572 amdgpu_dm_irq_set(adev, src, true); 573 } 574 575 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 576 } 577 EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_early); 578 579 void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev) 580 { 581 struct drm_device *dev = adev_to_drm(adev); 582 int src; 583 struct list_head *hnd_list_h, *hnd_list_l; 584 unsigned long irq_table_flags; 585 586 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 587 588 drm_dbg(adev_to_drm(adev), "DM_IRQ: resume\n"); 589 590 /** 591 * Renable HW interrupt for HPD and only since FLIP and VBLANK 592 * will be enabled from manage_dm_interrupts on enable CRTC. 593 */ 594 for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6; src++) { 595 hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; 596 hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; 597 if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) 598 amdgpu_dm_irq_set(adev, src, true); 599 } 600 601 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 602 603 if (dev->mode_config.poll_enabled) 604 drm_kms_helper_poll_enable(dev); 605 } 606 EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_late); 607 608 /* 609 * amdgpu_dm_irq_schedule_work - schedule all work items registered for the 610 * "irq_source". 611 */ 612 STATIC_IFN_KUNIT void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev, 613 enum dc_irq_source irq_source) 614 { 615 struct list_head *handler_list = &adev->dm.irq_handler_list_low_tab[irq_source]; 616 struct amdgpu_dm_irq_handler_data *handler_data; 617 bool work_queued = false; 618 unsigned long irq_table_flags; 619 620 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 621 622 if (!adev->dm.irq_wq) 623 goto out_unlock; 624 625 if (list_empty(handler_list)) 626 goto out_unlock; 627 628 list_for_each_entry(handler_data, handler_list, list) { 629 if (queue_work(adev->dm.irq_wq, &handler_data->work)) { 630 work_queued = true; 631 break; 632 } 633 } 634 635 if (!work_queued) { 636 struct amdgpu_dm_irq_handler_data *handler_data_add; 637 /*get the amdgpu_dm_irq_handler_data of first item pointed by handler_list*/ 638 handler_data = container_of(handler_list->next, struct amdgpu_dm_irq_handler_data, list); 639 640 /*allocate a new amdgpu_dm_irq_handler_data*/ 641 handler_data_add = kzalloc_obj(*handler_data, GFP_ATOMIC); 642 if (!handler_data_add) { 643 DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n"); 644 goto out_unlock; 645 } 646 647 /*copy new amdgpu_dm_irq_handler_data members from handler_data*/ 648 handler_data_add->handler = handler_data->handler; 649 handler_data_add->handler_arg = handler_data->handler_arg; 650 handler_data_add->dm = handler_data->dm; 651 handler_data_add->irq_source = irq_source; 652 653 list_add_tail(&handler_data_add->list, handler_list); 654 655 INIT_WORK(&handler_data_add->work, dm_irq_work_func); 656 657 if (queue_work(adev->dm.irq_wq, &handler_data_add->work)) 658 DRM_DEBUG("Queued work for handling interrupt from " 659 "display for IRQ source %d\n", 660 irq_source); 661 else 662 DRM_ERROR("Failed to queue work for handling interrupt " 663 "from display for IRQ source %d\n", 664 irq_source); 665 } 666 667 out_unlock: 668 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 669 } 670 EXPORT_IF_KUNIT(amdgpu_dm_irq_schedule_work); 671 672 /* 673 * amdgpu_dm_irq_immediate_work 674 * Callback high irq work immediately, don't send to work queue 675 */ 676 STATIC_IFN_KUNIT void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev, 677 enum dc_irq_source irq_source) 678 { 679 struct amdgpu_dm_irq_handler_data *handler_data; 680 unsigned long irq_table_flags; 681 682 DM_IRQ_TABLE_LOCK(adev, irq_table_flags); 683 684 list_for_each_entry(handler_data, 685 &adev->dm.irq_handler_list_high_tab[irq_source], 686 list) { 687 /* Call a subcomponent which registered for immediate 688 * interrupt notification 689 */ 690 handler_data->handler(handler_data->handler_arg); 691 } 692 693 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); 694 } 695 EXPORT_IF_KUNIT(amdgpu_dm_irq_immediate_work); 696 697 bool amdgpu_dm_irq_set(struct amdgpu_device *adev, enum dc_irq_source src, 698 bool enable) 699 { 700 guard(spinlock_irqsave)(&adev->dm.irq_reg_lock); 701 702 return dc_interrupt_set(adev->dm.dc, src, enable); 703 } 704 EXPORT_IF_KUNIT(amdgpu_dm_irq_set); 705 706 void amdgpu_dm_irq_ack(struct amdgpu_device *adev, enum dc_irq_source src) 707 { 708 guard(spinlock_irqsave)(&adev->dm.irq_reg_lock); 709 710 dc_interrupt_ack(adev->dm.dc, src); 711 } 712 EXPORT_IF_KUNIT(amdgpu_dm_irq_ack); 713 714 /** 715 * amdgpu_dm_irq_handler - Generic DM IRQ handler 716 * @adev: amdgpu base driver device containing the DM device 717 * @source: Unused 718 * @entry: Data about the triggered interrupt 719 * 720 * Calls all registered high irq work immediately, and schedules work for low 721 * irq. The DM IRQ table is used to find the corresponding handlers. 722 */ 723 STATIC_IFN_KUNIT int amdgpu_dm_irq_handler(struct amdgpu_device *adev, 724 struct amdgpu_irq_src *source, 725 struct amdgpu_iv_entry *entry) 726 { 727 728 enum dc_irq_source src = 729 dc_interrupt_to_irq_source( 730 adev->dm.dc, 731 entry->src_id, 732 entry->src_data[0]); 733 734 if (!DAL_VALID_IRQ_SRC_NUM(src)) { 735 DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", src); 736 return 0; 737 } 738 739 amdgpu_dm_irq_ack(adev, src); 740 741 /* Call high irq work immediately */ 742 amdgpu_dm_irq_immediate_work(adev, src); 743 /*Schedule low_irq work */ 744 amdgpu_dm_irq_schedule_work(adev, src); 745 746 return 0; 747 } 748 EXPORT_IF_KUNIT(amdgpu_dm_irq_handler); 749 750 STATIC_IFN_KUNIT enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type) 751 { 752 switch (type) { 753 case AMDGPU_HPD_1: 754 return DC_IRQ_SOURCE_HPD1; 755 case AMDGPU_HPD_2: 756 return DC_IRQ_SOURCE_HPD2; 757 case AMDGPU_HPD_3: 758 return DC_IRQ_SOURCE_HPD3; 759 case AMDGPU_HPD_4: 760 return DC_IRQ_SOURCE_HPD4; 761 case AMDGPU_HPD_5: 762 return DC_IRQ_SOURCE_HPD5; 763 case AMDGPU_HPD_6: 764 return DC_IRQ_SOURCE_HPD6; 765 default: 766 return DC_IRQ_SOURCE_INVALID; 767 } 768 } 769 EXPORT_IF_KUNIT(amdgpu_dm_hpd_to_dal_irq_source); 770 771 STATIC_IFN_KUNIT int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev, 772 struct amdgpu_irq_src *source, 773 unsigned int type, 774 enum amdgpu_interrupt_state state) 775 { 776 enum dc_irq_source src = amdgpu_dm_hpd_to_dal_irq_source(type); 777 bool st = (state == AMDGPU_IRQ_STATE_ENABLE); 778 779 amdgpu_dm_irq_set(adev, src, st); 780 return 0; 781 } 782 EXPORT_IF_KUNIT(amdgpu_dm_set_hpd_irq_state); 783 784 static inline int dm_irq_state(struct amdgpu_device *adev, 785 struct amdgpu_irq_src *source, 786 unsigned int crtc_id, 787 enum amdgpu_interrupt_state state, 788 const enum irq_type dal_irq_type, 789 const char *func) 790 { 791 bool st; 792 enum dc_irq_source irq_source; 793 struct dc *dc = adev->dm.dc; 794 struct amdgpu_crtc *acrtc = adev->mode_info.crtcs[crtc_id]; 795 796 if (!acrtc) { 797 DRM_ERROR( 798 "%s: crtc is NULL at id :%d\n", 799 func, 800 crtc_id); 801 return 0; 802 } 803 804 if (acrtc->otg_inst == -1) 805 return 0; 806 807 irq_source = dal_irq_type + acrtc->otg_inst; 808 809 st = (state == AMDGPU_IRQ_STATE_ENABLE); 810 811 if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed) 812 dc_allow_idle_optimizations(dc, false); 813 814 amdgpu_dm_irq_set(adev, irq_source, st); 815 return 0; 816 } 817 818 STATIC_IFN_KUNIT int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev, 819 struct amdgpu_irq_src *source, 820 unsigned int crtc_id, 821 enum amdgpu_interrupt_state state) 822 { 823 return dm_irq_state( 824 adev, 825 source, 826 crtc_id, 827 state, 828 IRQ_TYPE_PFLIP, 829 __func__); 830 } 831 EXPORT_IF_KUNIT(amdgpu_dm_set_pflip_irq_state); 832 833 STATIC_IFN_KUNIT int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev, 834 struct amdgpu_irq_src *source, 835 unsigned int crtc_id, 836 enum amdgpu_interrupt_state state) 837 { 838 return dm_irq_state( 839 adev, 840 source, 841 crtc_id, 842 state, 843 IRQ_TYPE_VBLANK, 844 __func__); 845 } 846 EXPORT_IF_KUNIT(amdgpu_dm_set_crtc_irq_state); 847 848 STATIC_IFN_KUNIT int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev, 849 struct amdgpu_irq_src *source, 850 unsigned int crtc_id, 851 enum amdgpu_interrupt_state state) 852 { 853 return dm_irq_state( 854 adev, 855 source, 856 crtc_id, 857 state, 858 IRQ_TYPE_VLINE0, 859 __func__); 860 } 861 EXPORT_IF_KUNIT(amdgpu_dm_set_vline0_irq_state); 862 863 STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev, 864 struct amdgpu_irq_src *source, 865 unsigned int crtc_id, 866 enum amdgpu_interrupt_state state) 867 { 868 enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX; 869 bool st = (state == AMDGPU_IRQ_STATE_ENABLE); 870 871 amdgpu_dm_irq_set(adev, irq_source, st); 872 return 0; 873 } 874 EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_outbox_irq_state); 875 876 STATIC_IFN_KUNIT int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev, 877 struct amdgpu_irq_src *source, 878 unsigned int crtc_id, 879 enum amdgpu_interrupt_state state) 880 { 881 return dm_irq_state( 882 adev, 883 source, 884 crtc_id, 885 state, 886 IRQ_TYPE_VUPDATE, 887 __func__); 888 } 889 EXPORT_IF_KUNIT(amdgpu_dm_set_vupdate_irq_state); 890 891 STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev, 892 struct amdgpu_irq_src *source, 893 unsigned int type, 894 enum amdgpu_interrupt_state state) 895 { 896 enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0; 897 bool st = (state == AMDGPU_IRQ_STATE_ENABLE); 898 899 amdgpu_dm_irq_set(adev, irq_source, st); 900 return 0; 901 } 902 EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_trace_irq_state); 903 904 static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = { 905 .set = amdgpu_dm_set_crtc_irq_state, 906 .process = amdgpu_dm_irq_handler, 907 }; 908 909 static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = { 910 .set = amdgpu_dm_set_vline0_irq_state, 911 .process = amdgpu_dm_irq_handler, 912 }; 913 914 static const struct amdgpu_irq_src_funcs dm_dmub_outbox_irq_funcs = { 915 .set = amdgpu_dm_set_dmub_outbox_irq_state, 916 .process = amdgpu_dm_irq_handler, 917 }; 918 919 static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = { 920 .set = amdgpu_dm_set_vupdate_irq_state, 921 .process = amdgpu_dm_irq_handler, 922 }; 923 924 static const struct amdgpu_irq_src_funcs dm_dmub_trace_irq_funcs = { 925 .set = amdgpu_dm_set_dmub_trace_irq_state, 926 .process = amdgpu_dm_irq_handler, 927 }; 928 929 static const struct amdgpu_irq_src_funcs dm_pageflip_irq_funcs = { 930 .set = amdgpu_dm_set_pflip_irq_state, 931 .process = amdgpu_dm_irq_handler, 932 }; 933 934 static const struct amdgpu_irq_src_funcs dm_hpd_irq_funcs = { 935 .set = amdgpu_dm_set_hpd_irq_state, 936 .process = amdgpu_dm_irq_handler, 937 }; 938 939 void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) 940 { 941 adev->crtc_irq.num_types = adev->mode_info.num_crtc; 942 adev->crtc_irq.funcs = &dm_crtc_irq_funcs; 943 944 adev->vline0_irq.num_types = adev->mode_info.num_crtc; 945 adev->vline0_irq.funcs = &dm_vline0_irq_funcs; 946 947 adev->dmub_outbox_irq.num_types = 1; 948 adev->dmub_outbox_irq.funcs = &dm_dmub_outbox_irq_funcs; 949 950 adev->vupdate_irq.num_types = adev->mode_info.num_crtc; 951 adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs; 952 953 adev->dmub_trace_irq.num_types = 1; 954 adev->dmub_trace_irq.funcs = &dm_dmub_trace_irq_funcs; 955 956 adev->pageflip_irq.num_types = adev->mode_info.num_crtc; 957 adev->pageflip_irq.funcs = &dm_pageflip_irq_funcs; 958 959 adev->hpd_irq.num_types = adev->mode_info.num_hpd; 960 adev->hpd_irq.funcs = &dm_hpd_irq_funcs; 961 } 962 EXPORT_IF_KUNIT(amdgpu_dm_set_irq_funcs); 963 964 void amdgpu_dm_outbox_init(struct amdgpu_device *adev) 965 { 966 amdgpu_dm_irq_set(adev, DC_IRQ_SOURCE_DMCUB_OUTBOX, true); 967 } 968 EXPORT_IF_KUNIT(amdgpu_dm_outbox_init); 969 970 /** 971 * amdgpu_dm_hpd_init - hpd setup callback. 972 * 973 * @adev: amdgpu_device pointer 974 * 975 * Setup the hpd pins used by the card (evergreen+). 976 * Enable the pin, set the polarity, and enable the hpd interrupts. 977 */ 978 void amdgpu_dm_hpd_init(struct amdgpu_device *adev) 979 { 980 struct drm_device *dev = adev_to_drm(adev); 981 struct drm_connector *connector; 982 struct drm_connector_list_iter iter; 983 int irq_type; 984 int i; 985 bool use_polling = false; 986 987 /* 988 * First, clear all hpd and hpdrx interrupts, over the links this ASIC 989 * has rather than over the whole enum: the unused slots of the DC 990 * interrupt table hold dummy handlers that assert when touched. 991 */ 992 for (i = 0; i < adev->dm.dc->link_count; i++) { 993 const struct dc_link *link = adev->dm.dc->links[i]; 994 995 if (link->irq_source_hpd != DC_IRQ_SOURCE_INVALID && 996 !amdgpu_dm_irq_set(adev, link->irq_source_hpd, false)) 997 drm_err(dev, "Failed to clear hpd source=%d on init\n", 998 link->irq_source_hpd); 999 1000 if (link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID && 1001 !amdgpu_dm_irq_set(adev, link->irq_source_hpd_rx, false)) 1002 drm_err(dev, "Failed to clear hpdrx source=%d on init\n", 1003 link->irq_source_hpd_rx); 1004 } 1005 1006 drm_connector_list_iter_begin(dev, &iter); 1007 drm_for_each_connector_iter(connector, &iter) { 1008 struct amdgpu_dm_connector *amdgpu_dm_connector; 1009 const struct dc_link *dc_link; 1010 1011 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) 1012 continue; 1013 1014 amdgpu_dm_connector = to_amdgpu_dm_connector(connector); 1015 dc_link = amdgpu_dm_connector->dc_link; 1016 if (!dc_link) 1017 continue; 1018 1019 /* 1020 * Analog connectors may be hot-plugged unlike other connector 1021 * types that don't support HPD. Only poll analog connectors. 1022 */ 1023 use_polling |= dc_connector_supports_analog(dc_link->link_id.id); 1024 1025 /* 1026 * Get a base driver irq reference for hpd ints for the lifetime 1027 * of dm. Note that only hpd interrupt types are registered with 1028 * base driver; hpd_rx types aren't. IOW, amdgpu_irq_get/put on 1029 * hpd_rx isn't available. DM currently controls hpd_rx 1030 * explicitly with amdgpu_dm_irq_set() 1031 */ 1032 if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { 1033 irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1; 1034 /* 1035 * TODO: There's a mismatch between mode_info.num_hpd 1036 * and what bios reports as the # of connectors with hpd 1037 * sources. Since the # of hpd source types registered 1038 * with base driver == mode_info.num_hpd, we have to 1039 * fallback to amdgpu_dm_irq_set for the remaining types. 1040 */ 1041 if (irq_type < adev->mode_info.num_hpd) { 1042 if (amdgpu_irq_get(adev, &adev->hpd_irq, irq_type)) 1043 drm_err(dev, "DM_IRQ: Failed get HPD for source=%d)!\n", 1044 dc_link->irq_source_hpd); 1045 } else { 1046 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd, 1047 true); 1048 } 1049 } 1050 1051 if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { 1052 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx, 1053 true); 1054 } 1055 } 1056 drm_connector_list_iter_end(&iter); 1057 1058 if (use_polling) 1059 drm_kms_helper_poll_init(dev); 1060 } 1061 EXPORT_IF_KUNIT(amdgpu_dm_hpd_init); 1062 1063 /** 1064 * amdgpu_dm_hpd_fini - hpd tear down callback. 1065 * 1066 * @adev: amdgpu_device pointer 1067 * 1068 * Tear down the hpd pins used by the card (evergreen+). 1069 * Disable the hpd interrupts. 1070 */ 1071 void amdgpu_dm_hpd_fini(struct amdgpu_device *adev) 1072 { 1073 struct drm_device *dev = adev_to_drm(adev); 1074 struct drm_connector *connector; 1075 struct drm_connector_list_iter iter; 1076 int irq_type; 1077 1078 drm_connector_list_iter_begin(dev, &iter); 1079 drm_for_each_connector_iter(connector, &iter) { 1080 struct amdgpu_dm_connector *amdgpu_dm_connector; 1081 const struct dc_link *dc_link; 1082 1083 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) 1084 continue; 1085 1086 amdgpu_dm_connector = to_amdgpu_dm_connector(connector); 1087 dc_link = amdgpu_dm_connector->dc_link; 1088 1089 if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { 1090 irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1; 1091 1092 /* TODO: See same TODO in amdgpu_dm_hpd_init() */ 1093 if (irq_type < adev->mode_info.num_hpd) { 1094 if (amdgpu_irq_put(adev, &adev->hpd_irq, irq_type)) 1095 drm_err(dev, "DM_IRQ: Failed put HPD for source=%d!\n", 1096 dc_link->irq_source_hpd); 1097 } else { 1098 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd, 1099 false); 1100 } 1101 } 1102 1103 if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { 1104 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx, 1105 false); 1106 } 1107 } 1108 drm_connector_list_iter_end(&iter); 1109 1110 if (dev->mode_config.poll_enabled) 1111 drm_kms_helper_poll_fini(dev); 1112 } 1113 EXPORT_IF_KUNIT(amdgpu_dm_hpd_fini); 1114 1115 /* ========== HPD handling ========== */ 1116 static void force_connector_state( 1117 struct amdgpu_dm_connector *aconnector, 1118 enum drm_connector_force force_state) 1119 { 1120 struct drm_connector *connector = &aconnector->base; 1121 1122 mutex_lock(&connector->dev->mode_config.mutex); 1123 aconnector->base.force = force_state; 1124 mutex_unlock(&connector->dev->mode_config.mutex); 1125 1126 mutex_lock(&aconnector->hpd_lock); 1127 drm_kms_helper_connector_hotplug_event(connector); 1128 mutex_unlock(&aconnector->hpd_lock); 1129 } 1130 1131 STATIC_IFN_KUNIT void dm_handle_hpd_rx_offload_work(struct work_struct *work) 1132 { 1133 struct hpd_rx_irq_offload_work *offload_work; 1134 struct amdgpu_dm_connector *aconnector; 1135 struct dc_link *dc_link; 1136 struct amdgpu_device *adev; 1137 enum dc_connection_type new_connection_type = dc_connection_none; 1138 unsigned long flags; 1139 union test_response test_response; 1140 1141 memset(&test_response, 0, sizeof(test_response)); 1142 1143 offload_work = container_of(work, struct hpd_rx_irq_offload_work, work); 1144 aconnector = offload_work->offload_wq->aconnector; 1145 adev = offload_work->adev; 1146 1147 if (!aconnector) { 1148 drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work"); 1149 goto skip; 1150 } 1151 1152 dc_link = aconnector->dc_link; 1153 1154 mutex_lock(&aconnector->hpd_lock); 1155 if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) 1156 drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); 1157 mutex_unlock(&aconnector->hpd_lock); 1158 1159 if (new_connection_type == dc_connection_none) 1160 goto skip; 1161 1162 if (amdgpu_in_reset(adev)) 1163 goto skip; 1164 1165 if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || 1166 offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { 1167 dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT); 1168 spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); 1169 offload_work->offload_wq->is_handling_mst_msg_rdy_event = false; 1170 spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); 1171 goto skip; 1172 } 1173 1174 mutex_lock(&adev->dm.dc_lock); 1175 if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) { 1176 dc_link_dp_handle_automated_test(dc_link); 1177 1178 if (aconnector->timing_changed) { 1179 /* force connector disconnect and reconnect */ 1180 force_connector_state(aconnector, DRM_FORCE_OFF); 1181 msleep(100); 1182 force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED); 1183 } 1184 1185 test_response.bits.ACK = 1; 1186 1187 core_link_write_dpcd( 1188 dc_link, 1189 DP_TEST_RESPONSE, 1190 &test_response.raw, 1191 sizeof(test_response)); 1192 } else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) && 1193 dc_link_check_link_loss_status(dc_link, &offload_work->data) && 1194 dc_link_dp_allow_hpd_rx_irq(dc_link)) { 1195 /* offload_work->data is from handle_hpd_rx_irq-> 1196 * schedule_hpd_rx_offload_work.this is defer handle 1197 * for hpd short pulse. upon here, link status may be 1198 * changed, need get latest link status from dpcd 1199 * registers. if link status is good, skip run link 1200 * training again. 1201 */ 1202 union hpd_irq_data irq_data; 1203 1204 memset(&irq_data, 0, sizeof(irq_data)); 1205 1206 /* before dc_link_dp_handle_link_loss, allow new link lost handle 1207 * request be added to work queue if link lost at end of dc_link_ 1208 * dp_handle_link_loss 1209 */ 1210 spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); 1211 offload_work->offload_wq->is_handling_link_loss = false; 1212 spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); 1213 1214 if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) && 1215 dc_link_check_link_loss_status(dc_link, &irq_data)) 1216 dc_link_dp_handle_link_loss(dc_link); 1217 } 1218 mutex_unlock(&adev->dm.dc_lock); 1219 1220 skip: 1221 kfree(offload_work); 1222 1223 } 1224 EXPORT_IF_KUNIT(dm_handle_hpd_rx_offload_work); 1225 1226 struct hpd_rx_irq_offload_work_queue *amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device *adev) 1227 { 1228 struct dc *dc = adev->dm.dc; 1229 int max_caps = dc->caps.max_links; 1230 int i = 0; 1231 struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL; 1232 1233 hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps); 1234 1235 if (!hpd_rx_offload_wq) 1236 return NULL; 1237 1238 1239 for (i = 0; i < max_caps; i++) { 1240 hpd_rx_offload_wq[i].wq = 1241 create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq"); 1242 1243 if (hpd_rx_offload_wq[i].wq == NULL) { 1244 drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!"); 1245 goto out_err; 1246 } 1247 1248 spin_lock_init(&hpd_rx_offload_wq[i].offload_lock); 1249 } 1250 1251 return hpd_rx_offload_wq; 1252 1253 out_err: 1254 for (i = 0; i < max_caps; i++) { 1255 if (hpd_rx_offload_wq[i].wq) 1256 destroy_workqueue(hpd_rx_offload_wq[i].wq); 1257 } 1258 kfree(hpd_rx_offload_wq); 1259 return NULL; 1260 } 1261 EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_create_workqueue); 1262 1263 void amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm) 1264 { 1265 int i; 1266 1267 if (dm->hpd_rx_offload_wq) { 1268 for (i = 0; i < dm->dc->caps.max_links; i++) 1269 flush_workqueue(dm->hpd_rx_offload_wq[i].wq); 1270 } 1271 } 1272 EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_work_suspend); 1273 1274 STATIC_IFN_KUNIT bool are_sinks_equal(const struct dc_sink *sink1, const struct dc_sink *sink2) 1275 { 1276 if (!sink1 || !sink2) 1277 return false; 1278 if (sink1->sink_signal != sink2->sink_signal) 1279 return false; 1280 1281 if (sink1->dc_edid.length != sink2->dc_edid.length) 1282 return false; 1283 1284 if (memcmp(sink1->dc_edid.raw_edid, sink2->dc_edid.raw_edid, 1285 sink1->dc_edid.length) != 0) 1286 return false; 1287 return true; 1288 } 1289 EXPORT_IF_KUNIT(are_sinks_equal); 1290 1291 1292 /** 1293 * DOC: amdgpu_dm_hdmi_hpd_debounce_work 1294 * 1295 * HDMI HPD debounce delay in milliseconds. When an HDMI display toggles HPD 1296 * (such as during power save transitions), this delay determines how long to 1297 * wait before processing the HPD event. This allows distinguishing between a 1298 * physical unplug (>hdmi_hpd_debounce_delay) 1299 * and a spontaneous RX HPD toggle (<hdmi_hpd_debounce_delay). 1300 * 1301 * If the toggle is less than this delay, the driver compares sink capabilities 1302 * and permits a hotplug event if they changed. 1303 * 1304 * The default value of 1500ms was chosen based on experimental testing with 1305 * various monitors that exhibit spontaneous HPD toggling behavior. 1306 */ 1307 void amdgpu_dm_hdmi_hpd_debounce_work(struct work_struct *work) 1308 { 1309 struct amdgpu_dm_connector *aconnector = 1310 container_of(to_delayed_work(work), struct amdgpu_dm_connector, 1311 hdmi_hpd_debounce_work); 1312 struct drm_connector *connector = &aconnector->base; 1313 struct drm_device *dev = connector->dev; 1314 struct amdgpu_device *adev = drm_to_adev(dev); 1315 struct dc *dc = aconnector->dc_link->ctx->dc; 1316 bool fake_reconnect = false; 1317 bool reallow_idle = false; 1318 bool ret = false; 1319 1320 guard(mutex)(&aconnector->hpd_lock); 1321 1322 /* Re-detect the display */ 1323 scoped_guard(mutex, &adev->dm.dc_lock) { 1324 if (dc->caps.ips_support && dc->ctx->dmub_srv->idle_allowed) { 1325 dc_allow_idle_optimizations(dc, false); 1326 reallow_idle = true; 1327 } 1328 ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD); 1329 } 1330 1331 if (ret) { 1332 /* Apply workaround delay for certain panels */ 1333 amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink); 1334 /* Compare sinks to determine if this was a spontaneous HPD toggle */ 1335 if (are_sinks_equal(aconnector->dc_link->local_sink, aconnector->hdmi_prev_sink)) { 1336 /* 1337 * Sinks match - this was a spontaneous HDMI HPD toggle. 1338 */ 1339 drm_dbg_kms(dev, "HDMI HPD: Sink unchanged after debounce, internal re-enable\n"); 1340 fake_reconnect = true; 1341 } 1342 1343 /* Update connector state */ 1344 amdgpu_dm_update_connector_after_detect(aconnector); 1345 1346 drm_modeset_lock_all(dev); 1347 dm_restore_drm_connector_state(dev, connector); 1348 drm_modeset_unlock_all(dev); 1349 1350 /* Only notify OS if sink actually changed */ 1351 if (!fake_reconnect && aconnector->base.force == DRM_FORCE_UNSPECIFIED) 1352 drm_kms_helper_hotplug_event(dev); 1353 } 1354 1355 /* Release the cached sink reference */ 1356 if (aconnector->hdmi_prev_sink) { 1357 dc_sink_release(aconnector->hdmi_prev_sink); 1358 aconnector->hdmi_prev_sink = NULL; 1359 } 1360 1361 scoped_guard(mutex, &adev->dm.dc_lock) { 1362 if (reallow_idle && dc->caps.ips_support) 1363 dc_allow_idle_optimizations(dc, true); 1364 } 1365 } 1366 EXPORT_IF_KUNIT(amdgpu_dm_hdmi_hpd_debounce_work); 1367 1368 STATIC_IFN_KUNIT void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector, 1369 enum dc_detect_reason reason) 1370 { 1371 struct drm_connector *connector = &aconnector->base; 1372 struct drm_device *dev = connector->dev; 1373 enum dc_connection_type new_connection_type = dc_connection_none; 1374 struct amdgpu_device *adev = drm_to_adev(dev); 1375 struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state); 1376 struct dc *dc = aconnector->dc_link->ctx->dc; 1377 bool ret = false; 1378 bool debounce_required = false; 1379 1380 if (adev->dm.disable_hpd_irq) 1381 return; 1382 1383 /* 1384 * In case of failure or MST no need to update connector status or notify the OS 1385 * since (for MST case) MST does this in its own context. 1386 */ 1387 guard(mutex)(&aconnector->hpd_lock); 1388 1389 if (adev->dm.hdcp_workqueue) { 1390 hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index); 1391 dm_con_state->update_hdcp = true; 1392 } 1393 if (aconnector->fake_enable) 1394 aconnector->fake_enable = false; 1395 1396 aconnector->timing_changed = false; 1397 1398 if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type)) 1399 drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); 1400 1401 /* 1402 * Check for HDMI disconnect with debounce enabled. 1403 */ 1404 debounce_required = (aconnector->hdmi_hpd_debounce_delay_ms > 0 && 1405 dc_is_hdmi_signal(aconnector->dc_link->connector_signal) && 1406 new_connection_type == dc_connection_none && 1407 aconnector->dc_link->local_sink != NULL); 1408 1409 if (aconnector->base.force && new_connection_type == dc_connection_none) { 1410 amdgpu_dm_emulated_link_detect(aconnector->dc_link); 1411 1412 drm_modeset_lock_all(dev); 1413 dm_restore_drm_connector_state(dev, connector); 1414 drm_modeset_unlock_all(dev); 1415 1416 if (aconnector->base.force == DRM_FORCE_UNSPECIFIED || 1417 reason == DETECT_REASON_HPDRX) 1418 drm_kms_helper_connector_hotplug_event(connector); 1419 } else if (debounce_required) { 1420 /* 1421 * HDMI disconnect detected - schedule delayed work instead of 1422 * processing immediately. This allows us to coalesce spurious 1423 * HDMI signals from physical unplugs. 1424 */ 1425 drm_dbg_kms(dev, "HDMI HPD: Disconnect detected, scheduling debounce work (%u ms)\n", 1426 aconnector->hdmi_hpd_debounce_delay_ms); 1427 1428 /* Cache the current sink for later comparison */ 1429 if (aconnector->hdmi_prev_sink) 1430 dc_sink_release(aconnector->hdmi_prev_sink); 1431 aconnector->hdmi_prev_sink = aconnector->dc_link->local_sink; 1432 if (aconnector->hdmi_prev_sink) 1433 dc_sink_retain(aconnector->hdmi_prev_sink); 1434 1435 /* Schedule delayed detection. */ 1436 if (mod_delayed_work(system_percpu_wq, 1437 &aconnector->hdmi_hpd_debounce_work, 1438 msecs_to_jiffies(aconnector->hdmi_hpd_debounce_delay_ms))) 1439 drm_dbg_kms(dev, "HDMI HPD: Re-scheduled debounce work\n"); 1440 1441 } else { 1442 1443 /* If the aconnector->hdmi_hpd_debounce_work is scheduled, exit early */ 1444 if (delayed_work_pending(&aconnector->hdmi_hpd_debounce_work)) 1445 return; 1446 1447 scoped_guard(mutex, &adev->dm.dc_lock) { 1448 dc_exit_ips_for_hw_access(dc); 1449 ret = dc_link_detect(aconnector->dc_link, reason); 1450 } 1451 if (ret) { 1452 /* w/a delay for certain panels */ 1453 amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink); 1454 amdgpu_dm_update_connector_after_detect(aconnector); 1455 1456 drm_modeset_lock_all(dev); 1457 dm_restore_drm_connector_state(dev, connector); 1458 drm_modeset_unlock_all(dev); 1459 1460 if (aconnector->base.force == DRM_FORCE_UNSPECIFIED || 1461 reason == DETECT_REASON_HPDRX) 1462 drm_kms_helper_connector_hotplug_event(connector); 1463 } 1464 } 1465 } 1466 EXPORT_IF_KUNIT(handle_hpd_irq_helper); 1467 1468 STATIC_IFN_KUNIT void handle_hpd_irq(void *param) 1469 { 1470 struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; 1471 1472 handle_hpd_irq_helper(aconnector, DETECT_REASON_HPD); 1473 1474 } 1475 EXPORT_IF_KUNIT(handle_hpd_irq); 1476 1477 STATIC_IFN_KUNIT void schedule_hpd_rx_offload_work(struct amdgpu_device *adev, struct hpd_rx_irq_offload_work_queue *offload_wq, 1478 union hpd_irq_data hpd_irq_data) 1479 { 1480 struct hpd_rx_irq_offload_work *offload_work = kzalloc_obj(*offload_work); 1481 1482 if (!offload_work) { 1483 drm_err(adev_to_drm(adev), "Failed to allocate hpd_rx_irq_offload_work.\n"); 1484 return; 1485 } 1486 1487 INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work); 1488 offload_work->data = hpd_irq_data; 1489 offload_work->offload_wq = offload_wq; 1490 offload_work->adev = adev; 1491 1492 queue_work(offload_wq->wq, &offload_work->work); 1493 drm_dbg_kms(adev_to_drm(adev), "queue work to handle hpd_rx offload work"); 1494 } 1495 EXPORT_IF_KUNIT(schedule_hpd_rx_offload_work); 1496 1497 STATIC_IFN_KUNIT void handle_hpd_rx_irq(void *param) 1498 { 1499 struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; 1500 struct drm_connector *connector = &aconnector->base; 1501 struct drm_device *dev = connector->dev; 1502 struct dc_link *dc_link = aconnector->dc_link; 1503 bool is_mst_root_connector = aconnector->mst_mgr.mst_state; 1504 bool result = false; 1505 enum dc_connection_type new_connection_type = dc_connection_none; 1506 struct amdgpu_device *adev = drm_to_adev(dev); 1507 union hpd_irq_data hpd_irq_data; 1508 bool link_loss = false; 1509 bool has_left_work = false; 1510 int idx = dc_link->link_index; 1511 struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx]; 1512 struct dc *dc = aconnector->dc_link->ctx->dc; 1513 1514 memset(&hpd_irq_data, 0, sizeof(hpd_irq_data)); 1515 1516 if (adev->dm.disable_hpd_irq) 1517 return; 1518 1519 /* 1520 * TODO:Temporary add mutex to protect hpd interrupt not have a gpio 1521 * conflict, after implement i2c helper, this mutex should be 1522 * retired. 1523 */ 1524 mutex_lock(&aconnector->hpd_lock); 1525 1526 result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, 1527 &link_loss, true, &has_left_work); 1528 1529 if (!has_left_work) 1530 goto out; 1531 1532 if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) { 1533 schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data); 1534 goto out; 1535 } 1536 1537 if (dc_link_dp_allow_hpd_rx_irq(dc_link)) { 1538 if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || 1539 hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { 1540 bool skip = false; 1541 1542 /* 1543 * DOWN_REP_MSG_RDY is also handled by polling method 1544 * mgr->cbs->poll_hpd_irq() 1545 */ 1546 spin_lock(&offload_wq->offload_lock); 1547 skip = offload_wq->is_handling_mst_msg_rdy_event; 1548 1549 if (!skip) 1550 offload_wq->is_handling_mst_msg_rdy_event = true; 1551 1552 spin_unlock(&offload_wq->offload_lock); 1553 1554 if (!skip) 1555 schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data); 1556 1557 goto out; 1558 } 1559 1560 if (link_loss) { 1561 bool skip = false; 1562 1563 spin_lock(&offload_wq->offload_lock); 1564 skip = offload_wq->is_handling_link_loss; 1565 1566 if (!skip) 1567 offload_wq->is_handling_link_loss = true; 1568 1569 spin_unlock(&offload_wq->offload_lock); 1570 1571 if (!skip) 1572 schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data); 1573 1574 goto out; 1575 } 1576 } 1577 1578 out: 1579 if (result && !is_mst_root_connector) { 1580 /* Downstream Port status changed. */ 1581 if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) 1582 drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); 1583 1584 if (aconnector->base.force && new_connection_type == dc_connection_none) { 1585 amdgpu_dm_emulated_link_detect(dc_link); 1586 1587 if (aconnector->fake_enable) 1588 aconnector->fake_enable = false; 1589 1590 amdgpu_dm_update_connector_after_detect(aconnector); 1591 1592 1593 drm_modeset_lock_all(dev); 1594 dm_restore_drm_connector_state(dev, connector); 1595 drm_modeset_unlock_all(dev); 1596 1597 drm_kms_helper_connector_hotplug_event(connector); 1598 } else { 1599 bool ret = false; 1600 1601 mutex_lock(&adev->dm.dc_lock); 1602 dc_exit_ips_for_hw_access(dc); 1603 ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX); 1604 mutex_unlock(&adev->dm.dc_lock); 1605 1606 if (ret) { 1607 if (aconnector->fake_enable) 1608 aconnector->fake_enable = false; 1609 1610 amdgpu_dm_update_connector_after_detect(aconnector); 1611 1612 drm_modeset_lock_all(dev); 1613 dm_restore_drm_connector_state(dev, connector); 1614 drm_modeset_unlock_all(dev); 1615 1616 drm_kms_helper_connector_hotplug_event(connector); 1617 } 1618 } 1619 } 1620 if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) { 1621 if (adev->dm.hdcp_workqueue) 1622 hdcp_handle_cpirq(adev->dm.hdcp_workqueue, aconnector->base.index); 1623 } 1624 1625 if (dc_link->type != dc_connection_mst_branch) 1626 drm_dp_cec_irq(&aconnector->dm_dp_aux.aux); 1627 1628 mutex_unlock(&aconnector->hpd_lock); 1629 } 1630 EXPORT_IF_KUNIT(handle_hpd_rx_irq); 1631 1632 /** 1633 * dmub_hpd_callback - DMUB HPD interrupt processing callback. 1634 * @adev: amdgpu_device pointer 1635 * @notify: dmub notification structure 1636 * 1637 * Dmub Hpd interrupt processing callback. Gets displayindex through the 1638 * ink index and calls helper to do the processing. 1639 */ 1640 STATIC_IFN_KUNIT void dmub_hpd_callback(struct amdgpu_device *adev, 1641 struct dmub_notification *notify) 1642 { 1643 struct amdgpu_dm_connector *aconnector; 1644 struct amdgpu_dm_connector *hpd_aconnector = NULL; 1645 struct drm_connector *connector; 1646 struct drm_connector_list_iter iter; 1647 struct dc_link *link; 1648 u8 link_index = 0; 1649 struct drm_device *dev; 1650 1651 if (adev == NULL) 1652 return; 1653 1654 if (notify == NULL) { 1655 drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL"); 1656 return; 1657 } 1658 1659 if (notify->link_index > adev->dm.dc->link_count) { 1660 drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index); 1661 return; 1662 } 1663 1664 /* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */ 1665 if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) { 1666 drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n"); 1667 return; 1668 } 1669 1670 link_index = notify->link_index; 1671 link = adev->dm.dc->links[link_index]; 1672 dev = adev->dm.ddev; 1673 1674 drm_connector_list_iter_begin(dev, &iter); 1675 drm_for_each_connector_iter(connector, &iter) { 1676 1677 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) 1678 continue; 1679 1680 aconnector = to_amdgpu_dm_connector(connector); 1681 if (link && aconnector->dc_link == link) { 1682 if (notify->type == DMUB_NOTIFICATION_HPD) 1683 drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index); 1684 else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) 1685 drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index); 1686 else 1687 drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n", 1688 notify->type, link_index); 1689 1690 hpd_aconnector = aconnector; 1691 break; 1692 } 1693 } 1694 drm_connector_list_iter_end(&iter); 1695 1696 if (hpd_aconnector) { 1697 if (notify->type == DMUB_NOTIFICATION_HPD) { 1698 if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG)) 1699 drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index); 1700 handle_hpd_irq_helper(hpd_aconnector, DETECT_REASON_HPD); 1701 } else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) { 1702 handle_hpd_rx_irq(hpd_aconnector); 1703 } 1704 } 1705 } 1706 EXPORT_IF_KUNIT(dmub_hpd_callback); 1707 1708 /** 1709 * dmub_hpd_sense_callback - DMUB HPD sense processing callback. 1710 * @adev: amdgpu_device pointer 1711 * @notify: dmub notification structure 1712 * 1713 * HPD sense changes can occur during low power states and need to be 1714 * notified from firmware to driver. 1715 */ 1716 STATIC_IFN_KUNIT void dmub_hpd_sense_callback(struct amdgpu_device *adev, 1717 struct dmub_notification *notify) 1718 { 1719 drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n"); 1720 } 1721 EXPORT_IF_KUNIT(dmub_hpd_sense_callback); 1722 1723 int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev) 1724 { 1725 struct drm_device *dev = adev_to_drm(adev); 1726 struct drm_connector *connector; 1727 struct amdgpu_dm_connector *aconnector; 1728 const struct dc_link *dc_link; 1729 struct dc_interrupt_params int_params = {0}; 1730 1731 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; 1732 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; 1733 1734 if (dc_is_dmub_outbox_supported(adev->dm.dc)) { 1735 if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD, 1736 dmub_hpd_callback, true)) { 1737 drm_err(adev_to_drm(adev), "fail to register dmub hpd callback"); 1738 return -EINVAL; 1739 } 1740 1741 if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ, 1742 dmub_hpd_callback, true)) { 1743 drm_err(adev_to_drm(adev), "fail to register dmub hpd callback"); 1744 return -EINVAL; 1745 } 1746 1747 if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_SENSE_NOTIFY, 1748 dmub_hpd_sense_callback, true)) { 1749 drm_err(adev_to_drm(adev), "fail to register dmub hpd sense callback"); 1750 return -EINVAL; 1751 } 1752 } 1753 1754 list_for_each_entry(connector, 1755 &dev->mode_config.connector_list, head) { 1756 1757 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) 1758 continue; 1759 1760 aconnector = to_amdgpu_dm_connector(connector); 1761 dc_link = aconnector->dc_link; 1762 1763 if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { 1764 int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; 1765 int_params.irq_source = dc_link->irq_source_hpd; 1766 1767 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 1768 int_params.irq_source < DC_IRQ_SOURCE_HPD1 || 1769 int_params.irq_source > DC_IRQ_SOURCE_HPD6) { 1770 drm_err(adev_to_drm(adev), 1771 "Failed to register hpd irq %u for %s!\n", 1772 int_params.irq_source, connector->name); 1773 return -EINVAL; 1774 } 1775 1776 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 1777 handle_hpd_irq, (void *) aconnector)) 1778 return -ENOMEM; 1779 } 1780 1781 if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { 1782 1783 /* Also register for DP short pulse (hpd_rx). */ 1784 int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; 1785 int_params.irq_source = dc_link->irq_source_hpd_rx; 1786 1787 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 1788 int_params.irq_source < DC_IRQ_SOURCE_HPD1RX || 1789 int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) { 1790 drm_err(adev_to_drm(adev), 1791 "Failed to register hpd rx irq %u for %s!\n", 1792 int_params.irq_source, connector->name); 1793 return -EINVAL; 1794 } 1795 1796 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 1797 handle_hpd_rx_irq, (void *) aconnector)) 1798 return -ENOMEM; 1799 } 1800 } 1801 return 0; 1802 } 1803 EXPORT_IF_KUNIT(amdgpu_dm_register_hpd_handlers); 1804 1805 /* ========== IRQ handlers ========== */ 1806 struct amdgpu_crtc * 1807 amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device *adev, 1808 int otg_inst) 1809 { 1810 struct drm_device *dev = adev_to_drm(adev); 1811 struct drm_crtc *crtc; 1812 struct amdgpu_crtc *amdgpu_crtc; 1813 1814 if (WARN_ON(otg_inst == -1)) 1815 return adev->mode_info.crtcs[0]; 1816 1817 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 1818 amdgpu_crtc = to_amdgpu_crtc(crtc); 1819 1820 if (amdgpu_crtc->otg_inst == otg_inst) 1821 return amdgpu_crtc; 1822 } 1823 1824 return NULL; 1825 } 1826 EXPORT_IF_KUNIT(amdgpu_dm_get_crtc_by_otg_inst); 1827 1828 /** 1829 * dm_pflip_high_irq() - Handle pageflip interrupt 1830 * @interrupt_params: ignored 1831 * 1832 * Handles the pageflip interrupt by notifying all interested parties 1833 * that the pageflip has been completed. 1834 */ 1835 STATIC_IFN_KUNIT void dm_pflip_high_irq(void *interrupt_params) 1836 { 1837 struct amdgpu_crtc *amdgpu_crtc; 1838 struct common_irq_params *irq_params = interrupt_params; 1839 struct amdgpu_device *adev = irq_params->adev; 1840 struct drm_device *dev = adev_to_drm(adev); 1841 unsigned long flags; 1842 struct drm_pending_vblank_event *e; 1843 u32 vpos, hpos, v_blank_start, v_blank_end; 1844 bool vrr_active; 1845 1846 amdgpu_crtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP); 1847 1848 /* IRQ could occur when in initial stage */ 1849 /* TODO work and BO cleanup */ 1850 if (amdgpu_crtc == NULL) { 1851 drm_dbg_state(dev, "CRTC is null, returning.\n"); 1852 return; 1853 } 1854 1855 spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); 1856 1857 if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { 1858 drm_dbg_state(dev, 1859 "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n", 1860 amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED, 1861 amdgpu_crtc->crtc_id, amdgpu_crtc); 1862 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); 1863 return; 1864 } 1865 1866 /* page flip completed. */ 1867 e = amdgpu_crtc->event; 1868 amdgpu_crtc->event = NULL; 1869 1870 WARN_ON(!e); 1871 1872 vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc); 1873 1874 /* Fixed refresh rate, or VRR scanout position outside front-porch? */ 1875 if (!vrr_active || 1876 !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start, 1877 &v_blank_end, &hpos, &vpos) || 1878 (vpos < v_blank_start)) { 1879 /* Update to correct count and vblank timestamp if racing with 1880 * vblank irq. This also updates to the correct vblank timestamp 1881 * even in VRR mode, as scanout is past the front-porch atm. 1882 */ 1883 drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); 1884 1885 /* Wake up userspace by sending the pageflip event with proper 1886 * count and timestamp of vblank of flip completion. 1887 */ 1888 if (e) { 1889 drm_crtc_send_vblank_event(&amdgpu_crtc->base, e); 1890 1891 /* Event sent, so done with vblank for this flip */ 1892 drm_crtc_vblank_put(&amdgpu_crtc->base); 1893 } 1894 } else if (e) { 1895 /* VRR active and inside front-porch: vblank count and 1896 * timestamp for pageflip event will only be up to date after 1897 * drm_crtc_handle_vblank() has been executed from late vblank 1898 * irq handler after start of back-porch (vline 0). We queue the 1899 * pageflip event for send-out by drm_crtc_handle_vblank() with 1900 * updated timestamp and count, once it runs after us. 1901 * 1902 * We need to open-code this instead of using the helper 1903 * drm_crtc_arm_vblank_event(), as that helper would 1904 * call drm_crtc_accurate_vblank_count(), which we must 1905 * not call in VRR mode while we are in front-porch! 1906 */ 1907 1908 /* sequence will be replaced by real count during send-out. */ 1909 e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base); 1910 e->pipe = amdgpu_crtc->crtc_id; 1911 1912 list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list); 1913 e = NULL; 1914 } 1915 1916 /* Keep track of vblank of this flip for flip throttling. We use the 1917 * cooked hw counter, as that one incremented at start of this vblank 1918 * of pageflip completion, so last_flip_vblank is the forbidden count 1919 * for queueing new pageflips if vsync + VRR is enabled. 1920 */ 1921 amdgpu_crtc->dm_irq_params.last_flip_vblank = 1922 amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base); 1923 1924 amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; 1925 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); 1926 1927 drm_dbg_state(dev, 1928 "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n", 1929 amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e); 1930 } 1931 EXPORT_IF_KUNIT(dm_pflip_high_irq); 1932 1933 STATIC_IFN_KUNIT void dm_handle_vmin_vmax_update(struct work_struct *offload_work) 1934 { 1935 struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work); 1936 struct amdgpu_device *adev = work->adev; 1937 struct dc_stream_state *stream = work->stream; 1938 struct dc_crtc_timing_adjust *adjust = work->adjust; 1939 1940 mutex_lock(&adev->dm.dc_lock); 1941 dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust); 1942 mutex_unlock(&adev->dm.dc_lock); 1943 1944 dc_stream_release(stream); 1945 kfree(work->adjust); 1946 kfree(work); 1947 } 1948 EXPORT_IF_KUNIT(dm_handle_vmin_vmax_update); 1949 1950 static void schedule_dc_vmin_vmax(struct amdgpu_device *adev, 1951 struct dc_stream_state *stream, 1952 struct dc_crtc_timing_adjust *adjust) 1953 { 1954 struct vupdate_offload_work *offload_work = kzalloc_obj(*offload_work, 1955 GFP_NOWAIT); 1956 if (!offload_work) { 1957 drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n"); 1958 return; 1959 } 1960 1961 struct dc_crtc_timing_adjust *adjust_copy = kzalloc_obj(*adjust_copy, 1962 GFP_NOWAIT); 1963 if (!adjust_copy) { 1964 drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n"); 1965 kfree(offload_work); 1966 return; 1967 } 1968 1969 dc_stream_retain(stream); 1970 memcpy(adjust_copy, adjust, sizeof(*adjust_copy)); 1971 1972 INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update); 1973 offload_work->adev = adev; 1974 offload_work->stream = stream; 1975 offload_work->adjust = adjust_copy; 1976 1977 queue_work(adev->dm.vmin_vmax_wq, &offload_work->work); 1978 } 1979 1980 /** 1981 * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling 1982 * @adev: amdgpu device 1983 * @acrtc: the CRTC to service 1984 * Performs writeback completion, vblank event handling, CRC processing, VRR BTR 1985 * updates and pageflip completion delivery. 1986 * 1987 * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from 1988 * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank 1989 * from dm_crtc_high_irq(). 1990 */ 1991 static void dm_crtc_high_irq_handler(struct amdgpu_device *adev, 1992 struct amdgpu_crtc *acrtc) 1993 { 1994 unsigned long flags; 1995 int vrr_active; 1996 bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0; 1997 1998 if (acrtc->wb_conn && acrtc->wb_pending) { 1999 if (acrtc->wb_frame_done) { 2000 /* 2001 * Second vblank: the DMA for the captured frame has 2002 * had a full frame period to flush to memory. Signal 2003 * the out fence now. 2004 */ 2005 amdgpu_dm_crtc_complete_writeback(acrtc); 2006 } else { 2007 /* 2008 * First vblank after arming: the frame has been 2009 * scanned out and the DMA is finishing. Disable 2010 * writeback immediately to prevent the hardware from 2011 * starting a new capture that would overwrite the 2012 * buffer. Signal completion on the next vblank to 2013 * ensure the DMA is fully flushed to memory. 2014 */ 2015 dc_stream_fc_disable_writeback(adev->dm.dc, 2016 acrtc->dm_irq_params.stream, 0); 2017 acrtc->wb_frame_done = true; 2018 } 2019 } 2020 2021 vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); 2022 2023 drm_dbg_vbl(adev_to_drm(adev), 2024 "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id, 2025 vrr_active, acrtc->dm_irq_params.active_planes); 2026 2027 /** 2028 * Core vblank handling. 2029 * 2030 * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch 2031 * point, which is the correct place to timestamp both VRR and non-VRR 2032 * vblanks. 2033 * 2034 * On DCE this handler runs at the start of front-porch, where only 2035 * non-VRR timestamping is valid; VRR vblank is deferred to 2036 * dm_vupdate_high_irq() after end of front-porch. 2037 */ 2038 if (is_dcn || !vrr_active) 2039 amdgpu_dm_crtc_handle_vblank(acrtc); 2040 2041 /** 2042 * Following stuff must happen at start of vblank, for crc 2043 * computation and below-the-range btr support in vrr mode. 2044 */ 2045 amdgpu_dm_crtc_handle_crc_irq(&acrtc->base); 2046 2047 /* BTR updates need to happen before VUPDATE on Vega and above. */ 2048 if (adev->family < AMDGPU_FAMILY_AI) 2049 return; 2050 2051 spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); 2052 2053 if (acrtc->dm_irq_params.stream && 2054 acrtc->dm_irq_params.vrr_params.supported) { 2055 bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; 2056 bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; 2057 bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; 2058 2059 mod_freesync_handle_v_update(adev->dm.freesync_module, 2060 acrtc->dm_irq_params.stream, 2061 &acrtc->dm_irq_params.vrr_params); 2062 2063 /* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */ 2064 if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { 2065 schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream, 2066 &acrtc->dm_irq_params.vrr_params.adjust); 2067 } 2068 } 2069 2070 /* 2071 * Deliver pageflip completion events (DCN only). 2072 * 2073 * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch 2074 * point. Deliver any pending pageflip completion event from here, 2075 * once HW has consumed the new address (the OTG no longer reports a 2076 * pending flip). 2077 * 2078 * Also handle the case here where there aren't any active planes and 2079 * DCN HUBP may be clock-gated, so the flip-pending status may be 2080 * undefined. 2081 */ 2082 if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && 2083 acrtc->event) { 2084 2085 if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) { 2086 drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); 2087 acrtc->event = NULL; 2088 drm_crtc_vblank_put(&acrtc->base); 2089 acrtc->pflip_status = AMDGPU_FLIP_NONE; 2090 } 2091 /* 2092 * If the flip is still pending, leave it armed and 2093 * retry on the next vupdate. 2094 */ 2095 } else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && 2096 acrtc->dm_irq_params.active_planes == 0) { 2097 2098 if (acrtc->event) { 2099 drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); 2100 acrtc->event = NULL; 2101 drm_crtc_vblank_put(&acrtc->base); 2102 } 2103 acrtc->pflip_status = AMDGPU_FLIP_NONE; 2104 } 2105 2106 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); 2107 } 2108 2109 STATIC_IFN_KUNIT void dm_vupdate_high_irq(void *interrupt_params) 2110 { 2111 struct common_irq_params *irq_params = interrupt_params; 2112 struct amdgpu_device *adev = irq_params->adev; 2113 struct amdgpu_crtc *acrtc; 2114 struct drm_device *drm_dev; 2115 struct drm_vblank_crtc *vblank; 2116 ktime_t frame_duration_ns, previous_timestamp; 2117 unsigned long flags; 2118 int vrr_active; 2119 2120 acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); 2121 if (!acrtc) 2122 return; 2123 2124 vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); 2125 drm_dev = acrtc->base.dev; 2126 vblank = drm_crtc_vblank_crtc(&acrtc->base); 2127 previous_timestamp = atomic64_read(&irq_params->previous_timestamp); 2128 frame_duration_ns = vblank->time - previous_timestamp; 2129 2130 if (frame_duration_ns > 0) { 2131 trace_amdgpu_refresh_rate_track(acrtc->base.index, 2132 frame_duration_ns, 2133 ktime_divns(NSEC_PER_SEC, frame_duration_ns)); 2134 atomic64_set(&irq_params->previous_timestamp, vblank->time); 2135 } 2136 2137 drm_dbg_vbl(drm_dev, 2138 "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, 2139 vrr_active); 2140 2141 /* 2142 * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver 2143 * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are 2144 * not used. Run the full handler here. 2145 */ 2146 if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) { 2147 dm_crtc_high_irq_handler(adev, acrtc); 2148 return; 2149 } 2150 2151 /* DCE only below. */ 2152 2153 /* Core vblank handling is done here after end of front-porch in 2154 * vrr mode, as vblank timestamping will give valid results 2155 * while now done after front-porch. This will also deliver 2156 * page-flip completion events that have been queued to us 2157 * if a pageflip happened inside front-porch. 2158 */ 2159 if (vrr_active && acrtc->dm_irq_params.stream) { 2160 bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; 2161 bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; 2162 bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state 2163 == VRR_STATE_ACTIVE_VARIABLE; 2164 2165 amdgpu_dm_crtc_handle_vblank(acrtc); 2166 2167 /* BTR processing for pre-DCE12 ASICs */ 2168 if (adev->family < AMDGPU_FAMILY_AI) { 2169 spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); 2170 mod_freesync_handle_v_update( 2171 adev->dm.freesync_module, 2172 acrtc->dm_irq_params.stream, 2173 &acrtc->dm_irq_params.vrr_params); 2174 2175 if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { 2176 schedule_dc_vmin_vmax(adev, 2177 acrtc->dm_irq_params.stream, 2178 &acrtc->dm_irq_params.vrr_params.adjust); 2179 } 2180 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); 2181 } 2182 } 2183 } 2184 EXPORT_IF_KUNIT(dm_vupdate_high_irq); 2185 2186 /** 2187 * dm_crtc_high_irq() - Handles CRTC interrupt 2188 * @interrupt_params: used for determining the CRTC instance 2189 * 2190 * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler. 2191 * 2192 * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is 2193 * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq(). 2194 */ 2195 STATIC_IFN_KUNIT void dm_crtc_high_irq(void *interrupt_params) 2196 { 2197 struct common_irq_params *irq_params = interrupt_params; 2198 struct amdgpu_device *adev = irq_params->adev; 2199 struct amdgpu_crtc *acrtc; 2200 2201 acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); 2202 if (!acrtc) 2203 return; 2204 2205 dm_crtc_high_irq_handler(adev, acrtc); 2206 } 2207 EXPORT_IF_KUNIT(dm_crtc_high_irq); 2208 2209 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) 2210 /** 2211 * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for 2212 * DCN generation ASICs 2213 * @interrupt_params: interrupt parameters 2214 * 2215 * Used to set crc window/read out crc value at vertical line 0 position 2216 */ 2217 static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) 2218 { 2219 struct common_irq_params *irq_params = interrupt_params; 2220 struct amdgpu_device *adev = irq_params->adev; 2221 struct amdgpu_crtc *acrtc; 2222 2223 acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0); 2224 2225 if (!acrtc) 2226 return; 2227 2228 amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base); 2229 } 2230 #endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */ 2231 2232 STATIC_IFN_KUNIT void dm_handle_hpd_work(struct work_struct *work) 2233 { 2234 struct dmub_hpd_work *dmub_hpd_wrk; 2235 2236 dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work); 2237 2238 if (!dmub_hpd_wrk->dmub_notify) { 2239 drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL"); 2240 return; 2241 } 2242 2243 if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) { 2244 dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev, 2245 dmub_hpd_wrk->dmub_notify); 2246 } 2247 2248 kfree(dmub_hpd_wrk->dmub_notify); 2249 kfree(dmub_hpd_wrk); 2250 2251 } 2252 EXPORT_IF_KUNIT(dm_handle_hpd_work); 2253 2254 STATIC_IFN_KUNIT const char *dmub_notification_type_str(enum dmub_notification_type e) 2255 { 2256 switch (e) { 2257 case DMUB_NOTIFICATION_NO_DATA: 2258 return "NO_DATA"; 2259 case DMUB_NOTIFICATION_AUX_REPLY: 2260 return "AUX_REPLY"; 2261 case DMUB_NOTIFICATION_HPD: 2262 return "HPD"; 2263 case DMUB_NOTIFICATION_HPD_IRQ: 2264 return "HPD_IRQ"; 2265 case DMUB_NOTIFICATION_SET_CONFIG_REPLY: 2266 return "SET_CONFIG_REPLY"; 2267 case DMUB_NOTIFICATION_DPIA_NOTIFICATION: 2268 return "DPIA_NOTIFICATION"; 2269 case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY: 2270 return "HPD_SENSE_NOTIFY"; 2271 case DMUB_NOTIFICATION_FUSED_IO: 2272 return "FUSED_IO"; 2273 default: 2274 return "<unknown>"; 2275 } 2276 } 2277 EXPORT_IF_KUNIT(dmub_notification_type_str); 2278 2279 #define DMUB_TRACE_MAX_READ 64 2280 /** 2281 * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt 2282 * @interrupt_params: used for determining the Outbox instance 2283 * 2284 * Handles the Outbox Interrupt 2285 * event handler. 2286 */ 2287 STATIC_IFN_KUNIT void dm_dmub_outbox1_low_irq(void *interrupt_params) 2288 { 2289 struct dmub_notification notify = {0}; 2290 struct common_irq_params *irq_params = interrupt_params; 2291 struct amdgpu_device *adev = irq_params->adev; 2292 struct amdgpu_display_manager *dm = &adev->dm; 2293 struct dmcub_trace_buf_entry entry = { 0 }; 2294 u32 count = 0; 2295 struct dmub_hpd_work *dmub_hpd_wrk; 2296 2297 do { 2298 if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) { 2299 trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count, 2300 entry.param0, entry.param1); 2301 2302 drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n", 2303 entry.trace_code, entry.tick_count, entry.param0, entry.param1); 2304 } else 2305 break; 2306 2307 count++; 2308 2309 } while (count <= DMUB_TRACE_MAX_READ); 2310 2311 if (count > DMUB_TRACE_MAX_READ) 2312 drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ"); 2313 2314 if (dc_enable_dmub_notifications(adev->dm.dc) && 2315 irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) { 2316 2317 do { 2318 dc_stat_get_dmub_notification(adev->dm.dc, ¬ify); 2319 if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) { 2320 drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type); 2321 continue; 2322 } 2323 if (!dm->dmub_callback[notify.type]) { 2324 drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n", 2325 dmub_notification_type_str(notify.type)); 2326 continue; 2327 } 2328 if (dm->dmub_thread_offload[notify.type]) { 2329 dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk, 2330 GFP_ATOMIC); 2331 if (!dmub_hpd_wrk) { 2332 drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk"); 2333 return; 2334 } 2335 dmub_hpd_wrk->dmub_notify = kmemdup(¬ify, sizeof(struct dmub_notification), 2336 GFP_ATOMIC); 2337 if (!dmub_hpd_wrk->dmub_notify) { 2338 kfree(dmub_hpd_wrk); 2339 drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify"); 2340 return; 2341 } 2342 INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work); 2343 dmub_hpd_wrk->adev = adev; 2344 queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work); 2345 } else { 2346 dm->dmub_callback[notify.type](adev, ¬ify); 2347 } 2348 } while (notify.pending_notification); 2349 } 2350 } 2351 EXPORT_IF_KUNIT(dm_dmub_outbox1_low_irq); 2352 2353 /* Register IRQ sources and initialize IRQ callbacks */ 2354 int amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device *adev) 2355 { 2356 struct dc *dc = adev->dm.dc; 2357 struct common_irq_params *c_irq_params; 2358 struct dc_interrupt_params int_params = {0}; 2359 int r; 2360 int i; 2361 unsigned int src_id; 2362 unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY; 2363 /* Use different interrupts for VBLANK on DCE 6 vs. newer. */ 2364 const unsigned int vblank_d1 = 2365 adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0 2366 ? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1; 2367 2368 if (adev->family >= AMDGPU_FAMILY_AI) 2369 client_id = SOC15_IH_CLIENTID_DCE; 2370 2371 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; 2372 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; 2373 2374 /* 2375 * Actions of amdgpu_irq_add_id(): 2376 * 1. Register a set() function with base driver. 2377 * Base driver will call set() function to enable/disable an 2378 * interrupt in DC hardware. 2379 * 2. Register amdgpu_dm_irq_handler(). 2380 * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts 2381 * coming from DC hardware. 2382 * amdgpu_dm_irq_handler() will re-direct the interrupt to DC 2383 * for acknowledging and handling. 2384 */ 2385 2386 /* Use VBLANK interrupt */ 2387 for (i = 0; i < adev->mode_info.num_crtc; i++) { 2388 src_id = vblank_d1 + i; 2389 r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq); 2390 if (r) { 2391 drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n"); 2392 return r; 2393 } 2394 2395 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; 2396 int_params.irq_source = 2397 dc_interrupt_to_irq_source(dc, src_id, 0); 2398 2399 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 2400 int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || 2401 int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { 2402 drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n"); 2403 return -EINVAL; 2404 } 2405 2406 c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; 2407 2408 c_irq_params->adev = adev; 2409 c_irq_params->irq_src = int_params.irq_source; 2410 2411 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 2412 dm_crtc_high_irq, c_irq_params)) 2413 return -ENOMEM; 2414 } 2415 2416 if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) { 2417 /* Use VUPDATE interrupt */ 2418 for (i = 0; i < adev->mode_info.num_crtc; i++) { 2419 src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2; 2420 r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq); 2421 if (r) { 2422 drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n"); 2423 return r; 2424 } 2425 2426 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; 2427 int_params.irq_source = 2428 dc_interrupt_to_irq_source(dc, src_id, 0); 2429 2430 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 2431 int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || 2432 int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { 2433 drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n"); 2434 return -EINVAL; 2435 } 2436 2437 c_irq_params = &adev->dm.vupdate_params[ 2438 int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; 2439 c_irq_params->adev = adev; 2440 c_irq_params->irq_src = int_params.irq_source; 2441 2442 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 2443 dm_vupdate_high_irq, c_irq_params)) 2444 return -ENOMEM; 2445 } 2446 } 2447 2448 /* Use GRPH_PFLIP interrupt */ 2449 for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; 2450 i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { 2451 r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); 2452 if (r) { 2453 drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n"); 2454 return r; 2455 } 2456 2457 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; 2458 int_params.irq_source = 2459 dc_interrupt_to_irq_source(dc, i, 0); 2460 2461 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 2462 int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || 2463 int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { 2464 drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n"); 2465 return -EINVAL; 2466 } 2467 2468 c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; 2469 2470 c_irq_params->adev = adev; 2471 c_irq_params->irq_src = int_params.irq_source; 2472 2473 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 2474 dm_pflip_high_irq, c_irq_params)) 2475 return -ENOMEM; 2476 } 2477 2478 /* HPD */ 2479 r = amdgpu_irq_add_id(adev, client_id, 2480 VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); 2481 if (r) { 2482 drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n"); 2483 return r; 2484 } 2485 2486 r = amdgpu_dm_register_hpd_handlers(adev); 2487 2488 return r; 2489 } 2490 EXPORT_IF_KUNIT(amdgpu_dm_dce110_register_irq_handlers); 2491 2492 /* Register IRQ sources and initialize IRQ callbacks */ 2493 int amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device *adev) 2494 { 2495 struct dc *dc = adev->dm.dc; 2496 struct common_irq_params *c_irq_params; 2497 struct dc_interrupt_params int_params = {0}; 2498 int r; 2499 int i; 2500 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) 2501 static const unsigned int vrtl_int_srcid[] = { 2502 DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL, 2503 DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL, 2504 DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL, 2505 DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL, 2506 DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL, 2507 DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL 2508 }; 2509 #endif 2510 2511 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; 2512 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; 2513 2514 /* 2515 * Actions of amdgpu_irq_add_id(): 2516 * 1. Register a set() function with base driver. 2517 * Base driver will call set() function to enable/disable an 2518 * interrupt in DC hardware. 2519 * 2. Register amdgpu_dm_irq_handler(). 2520 * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts 2521 * coming from DC hardware. 2522 * amdgpu_dm_irq_handler() will re-direct the interrupt to DC 2523 * for acknowledging and handling. 2524 */ 2525 2526 /* Use otg vertical line interrupt */ 2527 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) 2528 for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) { 2529 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, 2530 vrtl_int_srcid[i], &adev->vline0_irq); 2531 2532 if (r) { 2533 drm_err(adev_to_drm(adev), "Failed to add vline0 irq id!\n"); 2534 return r; 2535 } 2536 2537 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; 2538 int_params.irq_source = 2539 dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0); 2540 2541 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 2542 int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 || 2543 int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) { 2544 drm_err(adev_to_drm(adev), "Failed to register vline0 irq!\n"); 2545 return -EINVAL; 2546 } 2547 2548 c_irq_params = &adev->dm.vline0_params[int_params.irq_source 2549 - DC_IRQ_SOURCE_DC1_VLINE0]; 2550 2551 c_irq_params->adev = adev; 2552 c_irq_params->irq_src = int_params.irq_source; 2553 2554 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 2555 dm_dcn_vertical_interrupt0_high_irq, 2556 c_irq_params)) 2557 return -ENOMEM; 2558 } 2559 #endif 2560 2561 /* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to 2562 * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx 2563 * to trigger at end of each vblank, regardless of state of the lock, 2564 * matching DCE behaviour. 2565 */ 2566 for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT; 2567 i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1; 2568 i++) { 2569 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq); 2570 2571 if (r) { 2572 drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n"); 2573 return r; 2574 } 2575 2576 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; 2577 int_params.irq_source = 2578 dc_interrupt_to_irq_source(dc, i, 0); 2579 2580 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || 2581 int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || 2582 int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { 2583 drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n"); 2584 return -EINVAL; 2585 } 2586 2587 c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; 2588 2589 c_irq_params->adev = adev; 2590 c_irq_params->irq_src = int_params.irq_source; 2591 2592 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 2593 dm_vupdate_high_irq, c_irq_params)) 2594 return -ENOMEM; 2595 } 2596 2597 /* HPD */ 2598 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT, 2599 &adev->hpd_irq); 2600 if (r) { 2601 drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n"); 2602 return r; 2603 } 2604 2605 r = amdgpu_dm_register_hpd_handlers(adev); 2606 2607 return r; 2608 } 2609 EXPORT_IF_KUNIT(amdgpu_dm_dcn10_register_irq_handlers); 2610 2611 /* Register Outbox IRQ sources and initialize IRQ callbacks */ 2612 int amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device *adev) 2613 { 2614 struct dc *dc = adev->dm.dc; 2615 struct common_irq_params *c_irq_params; 2616 struct dc_interrupt_params int_params = {0}; 2617 int r, i; 2618 2619 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; 2620 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; 2621 2622 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT, 2623 &adev->dmub_outbox_irq); 2624 if (r) { 2625 drm_err(adev_to_drm(adev), "Failed to add outbox irq id!\n"); 2626 return r; 2627 } 2628 2629 if (dc->ctx->dmub_srv) { 2630 i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT; 2631 int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; 2632 int_params.irq_source = 2633 dc_interrupt_to_irq_source(dc, i, 0); 2634 2635 c_irq_params = &adev->dm.dmub_outbox_params[0]; 2636 2637 c_irq_params->adev = adev; 2638 c_irq_params->irq_src = int_params.irq_source; 2639 2640 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, 2641 dm_dmub_outbox1_low_irq, c_irq_params)) 2642 return -ENOMEM; 2643 } 2644 2645 return 0; 2646 } 2647 EXPORT_IF_KUNIT(amdgpu_dm_register_outbox_irq_handlers); 2648