1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2015 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors: AMD
24 *
25 */
26
27 #include "dm_services_types.h"
28 #include "dc.h"
29 #include "dc/dc_dmub_srv.h"
30 #include "dc/dc_stat.h"
31
32 #include "amdgpu.h"
33 #include "amdgpu_display.h"
34 #include "amdgpu_dm.h"
35 #include "amdgpu_dm_irq.h"
36 #include "dm_helpers.h"
37 #include "amdgpu_dm_crtc.h"
38 #include "amdgpu_dm_hdcp.h"
39 #include "amdgpu_dm_mst_types.h"
40 #include "amdgpu_dm_dmub.h"
41 #include "amdgpu_dm_trace.h"
42 #include "link/protocols/link_dpcd.h"
43 #include "link_service_types.h"
44 #include "ivsrcid/ivsrcid_vislands30.h"
45 #include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
46 #include "modules/inc/mod_freesync.h"
47 #include <drm/drm_vblank.h>
48
49 /**
50 * DOC: overview
51 *
52 * DM provides another layer of IRQ management on top of what the base driver
53 * already provides. This is something that could be cleaned up, and is a
54 * future TODO item.
55 *
56 * The base driver provides IRQ source registration with DRM, handler
57 * registration into the base driver's IRQ table, and a handler callback
58 * amdgpu_irq_handler(), with which DRM calls on interrupts. This generic
59 * handler looks up the IRQ table, and calls the respective
60 * &amdgpu_irq_src_funcs.process hookups.
61 *
62 * What DM provides on top are two IRQ tables specifically for top-half and
63 * bottom-half IRQ handling, with the bottom-half implementing workqueues:
64 *
65 * - &amdgpu_display_manager.irq_handler_list_high_tab
66 * - &amdgpu_display_manager.irq_handler_list_low_tab
67 *
68 * They override the base driver's IRQ table, and the effect can be seen
69 * in the hooks that DM provides for &amdgpu_irq_src_funcs.process. They
70 * are all set to the DM generic handler amdgpu_dm_irq_handler(), which looks up
71 * DM's IRQ tables. However, in order for base driver to recognize this hook, DM
72 * still needs to register the IRQ with the base driver. See
73 * amdgpu_dm_dce110_register_irq_handlers() and
74 * amdgpu_dm_dcn10_register_irq_handlers().
75 *
76 * To expose DC's hardware interrupt toggle to the base driver, DM implements
77 * &amdgpu_irq_src_funcs.set hooks. Base driver calls it through
78 * amdgpu_irq_update() to enable or disable the interrupt.
79 */
80
81 /******************************************************************************
82 * Private declarations.
83 *****************************************************************************/
84
85 /**
86 * struct amdgpu_dm_irq_handler_data - Data for DM interrupt handlers.
87 *
88 * @list: Linked list entry referencing the next/previous handler
89 * @handler: Handler function
90 * @handler_arg: Argument passed to the handler when triggered
91 * @dm: DM which this handler belongs to
92 * @irq_source: DC interrupt source that this handler is registered for
93 * @work: work struct
94 */
95 struct amdgpu_dm_irq_handler_data {
96 struct list_head list;
97 interrupt_handler handler;
98 void *handler_arg;
99
100 struct amdgpu_display_manager *dm;
101 /* DAL irq source which registered for this interrupt. */
102 enum dc_irq_source irq_source;
103 struct work_struct work;
104 };
105
106 #define DM_IRQ_TABLE_LOCK(adev, flags) \
107 spin_lock_irqsave(&adev->dm.irq_handler_list_table_lock, flags)
108
109 #define DM_IRQ_TABLE_UNLOCK(adev, flags) \
110 spin_unlock_irqrestore(&adev->dm.irq_handler_list_table_lock, flags)
111
112 /******************************************************************************
113 * Private functions.
114 *****************************************************************************/
115
init_handler_common_data(struct amdgpu_dm_irq_handler_data * hcd,void (* ih)(void *),void * args,struct amdgpu_display_manager * dm)116 static void init_handler_common_data(struct amdgpu_dm_irq_handler_data *hcd,
117 void (*ih)(void *),
118 void *args,
119 struct amdgpu_display_manager *dm)
120 {
121 hcd->handler = ih;
122 hcd->handler_arg = args;
123 hcd->dm = dm;
124 }
125
126 /**
127 * dm_irq_work_func() - Handle an IRQ outside of the interrupt handler proper.
128 *
129 * @work: work struct
130 */
dm_irq_work_func(struct work_struct * work)131 static void dm_irq_work_func(struct work_struct *work)
132 {
133 struct amdgpu_dm_irq_handler_data *handler_data =
134 container_of(work, struct amdgpu_dm_irq_handler_data, work);
135
136 handler_data->handler(handler_data->handler_arg);
137
138 /* Call a DAL subcomponent which registered for interrupt notification
139 * at INTERRUPT_LOW_IRQ_CONTEXT.
140 * (The most common use is HPD interrupt)
141 */
142 }
143
144 /*
145 * Remove a handler and return a pointer to handler list from which the
146 * handler was removed.
147 */
remove_irq_handler(struct amdgpu_device * adev,void * ih,const struct dc_interrupt_params * int_params)148 static struct list_head *remove_irq_handler(struct amdgpu_device *adev,
149 void *ih,
150 const struct dc_interrupt_params *int_params)
151 {
152 struct list_head *hnd_list;
153 struct list_head *entry, *tmp;
154 struct amdgpu_dm_irq_handler_data *handler;
155 unsigned long irq_table_flags;
156 bool handler_removed = false;
157 enum dc_irq_source irq_source;
158
159 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
160
161 irq_source = int_params->irq_source;
162
163 switch (int_params->int_context) {
164 case INTERRUPT_HIGH_IRQ_CONTEXT:
165 hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source];
166 break;
167 case INTERRUPT_LOW_IRQ_CONTEXT:
168 default:
169 hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source];
170 break;
171 }
172
173 list_for_each_safe(entry, tmp, hnd_list) {
174
175 handler = list_entry(entry, struct amdgpu_dm_irq_handler_data,
176 list);
177
178 if (handler == NULL)
179 continue;
180
181 if (ih == handler->handler) {
182 /* Found our handler. Remove it from the list. */
183 list_del(&handler->list);
184 handler_removed = true;
185 break;
186 }
187 }
188
189 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
190
191 if (!handler_removed) {
192 /* Not necessarily an error - caller may not
193 * know the context.
194 */
195 return NULL;
196 }
197
198 if (int_params->int_context == INTERRUPT_LOW_IRQ_CONTEXT)
199 cancel_work_sync(&handler->work);
200
201 kfree(handler);
202
203 DRM_DEBUG_KMS(
204 "DM_IRQ: removed irq handler: %p for: dal_src=%d, irq context=%d\n",
205 ih, int_params->irq_source, int_params->int_context);
206
207 return hnd_list;
208 }
209
210 static bool
validate_irq_registration_params(struct dc_interrupt_params * int_params,void (* ih)(void *))211 validate_irq_registration_params(struct dc_interrupt_params *int_params,
212 void (*ih)(void *))
213 {
214 if (NULL == int_params || NULL == ih) {
215 DRM_ERROR("DM_IRQ: invalid input!\n");
216 return false;
217 }
218
219 if (int_params->int_context >= INTERRUPT_CONTEXT_NUMBER) {
220 DRM_ERROR("DM_IRQ: invalid context: %d!\n",
221 int_params->int_context);
222 return false;
223 }
224
225 if (!DAL_VALID_IRQ_SRC_NUM(int_params->irq_source)) {
226 DRM_ERROR("DM_IRQ: invalid irq_source: %d!\n",
227 int_params->irq_source);
228 return false;
229 }
230
231 return true;
232 }
233
validate_irq_unregistration_params(enum dc_irq_source irq_source,irq_handler_idx handler_idx)234 static bool validate_irq_unregistration_params(enum dc_irq_source irq_source,
235 irq_handler_idx handler_idx)
236 {
237 if (handler_idx == DAL_INVALID_IRQ_HANDLER_IDX) {
238 DRM_ERROR("DM_IRQ: invalid handler_idx==NULL!\n");
239 return false;
240 }
241
242 if (!DAL_VALID_IRQ_SRC_NUM(irq_source)) {
243 DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", irq_source);
244 return false;
245 }
246
247 return true;
248 }
249 /******************************************************************************
250 * Public functions.
251 *
252 * Note: caller is responsible for input validation.
253 *****************************************************************************/
254
255 /**
256 * amdgpu_dm_irq_register_interrupt() - Register a handler within DM.
257 * @adev: The base driver device containing the DM device.
258 * @int_params: Interrupt parameters containing the source, and handler context
259 * @ih: Function pointer to the interrupt handler to register
260 * @handler_args: Arguments passed to the handler when the interrupt occurs
261 *
262 * Register an interrupt handler for the given IRQ source, under the given
263 * context. The context can either be high or low. High context handlers are
264 * executed directly within ISR context, while low context is executed within a
265 * workqueue, thereby allowing operations that sleep.
266 *
267 * Registered handlers are called in a FIFO manner, i.e. the most recently
268 * registered handler will be called first.
269 *
270 * Return: Handler data &struct amdgpu_dm_irq_handler_data containing the IRQ
271 * source, handler function, and args
272 */
amdgpu_dm_irq_register_interrupt(struct amdgpu_device * adev,struct dc_interrupt_params * int_params,void (* ih)(void *),void * handler_args)273 void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev,
274 struct dc_interrupt_params *int_params,
275 void (*ih)(void *),
276 void *handler_args)
277 {
278 struct list_head *hnd_list;
279 struct amdgpu_dm_irq_handler_data *handler_data;
280 unsigned long irq_table_flags;
281 enum dc_irq_source irq_source;
282
283 if (!validate_irq_registration_params(int_params, ih))
284 return DAL_INVALID_IRQ_HANDLER_IDX;
285
286 handler_data = kzalloc_obj(*handler_data);
287 if (!handler_data) {
288 DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n");
289 return DAL_INVALID_IRQ_HANDLER_IDX;
290 }
291
292 init_handler_common_data(handler_data, ih, handler_args, &adev->dm);
293
294 irq_source = int_params->irq_source;
295
296 handler_data->irq_source = irq_source;
297
298 /* Lock the list, add the handler. */
299 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
300
301 switch (int_params->int_context) {
302 case INTERRUPT_HIGH_IRQ_CONTEXT:
303 hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source];
304 break;
305 case INTERRUPT_LOW_IRQ_CONTEXT:
306 default:
307 hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source];
308 INIT_WORK(&handler_data->work, dm_irq_work_func);
309 break;
310 }
311
312 list_add_tail(&handler_data->list, hnd_list);
313
314 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
315
316 /* This pointer will be stored by code which requested interrupt
317 * registration.
318 * The same pointer will be needed in order to unregister the
319 * interrupt.
320 */
321
322 DRM_DEBUG_KMS(
323 "DM_IRQ: added irq handler: %p for: dal_src=%d, irq context=%d\n",
324 handler_data,
325 irq_source,
326 int_params->int_context);
327
328 return handler_data;
329 }
330 EXPORT_IF_KUNIT(amdgpu_dm_irq_register_interrupt);
331
332 /**
333 * amdgpu_dm_irq_unregister_interrupt() - Remove a handler from the DM IRQ table
334 * @adev: The base driver device containing the DM device
335 * @irq_source: IRQ source to remove the given handler from
336 * @ih: Function pointer to the interrupt handler to unregister
337 *
338 * Go through both low and high context IRQ tables, and find the given handler
339 * for the given irq source. If found, remove it. Otherwise, do nothing.
340 */
amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device * adev,enum dc_irq_source irq_source,void * ih)341 void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev,
342 enum dc_irq_source irq_source,
343 void *ih)
344 {
345 struct list_head *handler_list;
346 struct dc_interrupt_params int_params;
347 int i;
348
349 if (!validate_irq_unregistration_params(irq_source, ih))
350 return;
351
352 memset(&int_params, 0, sizeof(int_params));
353
354 int_params.irq_source = irq_source;
355
356 for (i = 0; i < INTERRUPT_CONTEXT_NUMBER; i++) {
357
358 int_params.int_context = i;
359
360 handler_list = remove_irq_handler(adev, ih, &int_params);
361
362 if (handler_list != NULL)
363 break;
364 }
365
366 if (handler_list == NULL) {
367 /* If we got here, it means we searched all irq contexts
368 * for this irq source, but the handler was not found.
369 */
370 DRM_ERROR(
371 "DM_IRQ: failed to find irq handler:%p for irq_source:%d!\n",
372 ih, irq_source);
373 }
374 }
375 EXPORT_IF_KUNIT(amdgpu_dm_irq_unregister_interrupt);
376
377 /**
378 * amdgpu_dm_irq_init() - Initialize DM IRQ management
379 * @adev: The base driver device containing the DM device
380 *
381 * Initialize DM's high and low context IRQ tables.
382 *
383 * The N by M table contains N IRQ sources, with M
384 * &struct amdgpu_dm_irq_handler_data hooked together in a linked list. The
385 * list_heads are initialized here. When an interrupt n is triggered, all m
386 * handlers are called in sequence, FIFO according to registration order.
387 *
388 * The low context table requires special steps to initialize, since handlers
389 * will be deferred to a workqueue. See &struct irq_list_head.
390 */
amdgpu_dm_irq_init(struct amdgpu_device * adev)391 int amdgpu_dm_irq_init(struct amdgpu_device *adev)
392 {
393 int src;
394 struct list_head *lh;
395
396 DRM_DEBUG_KMS("DM_IRQ\n");
397
398 spin_lock_init(&adev->dm.irq_handler_list_table_lock);
399 spin_lock_init(&adev->dm.irq_reg_lock);
400
401 adev->dm.irq_wq = alloc_workqueue("amdgpu_dm_irq",
402 WQ_UNBOUND | WQ_HIGHPRI, 0);
403
404 if (!adev->dm.irq_wq)
405 return -ENOMEM;
406
407 adev->dm.vmin_vmax_wq = alloc_workqueue("amdgpu_dm_vmin_vmax",
408 WQ_UNBOUND, 0);
409
410 if (!adev->dm.vmin_vmax_wq) {
411 destroy_workqueue(adev->dm.irq_wq);
412 adev->dm.irq_wq = NULL;
413 return -ENOMEM;
414 }
415
416 for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
417 /* low context handler list init */
418 lh = &adev->dm.irq_handler_list_low_tab[src];
419 INIT_LIST_HEAD(lh);
420 /* high context handler init */
421 INIT_LIST_HEAD(&adev->dm.irq_handler_list_high_tab[src]);
422 }
423
424 return 0;
425 }
426 EXPORT_IF_KUNIT(amdgpu_dm_irq_init);
427
428 /**
429 * amdgpu_dm_irq_fini() - Tear down DM IRQ management
430 * @adev: The base driver device containing the DM device
431 *
432 * Removes all handlers from the IRQ tables under the spinlock, cancels
433 * pending work items, and deallocates all handler data.
434 */
amdgpu_dm_irq_fini(struct amdgpu_device * adev)435 void amdgpu_dm_irq_fini(struct amdgpu_device *adev)
436 {
437 int src;
438 LIST_HEAD(low_handlers);
439 LIST_HEAD(high_handlers);
440 struct list_head *entry, *tmp;
441 struct amdgpu_dm_irq_handler_data *handler;
442 unsigned long irq_table_flags;
443
444 DRM_DEBUG_KMS("DM_IRQ: releasing resources.\n");
445
446 for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
447 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
448
449 /*
450 * Move all handlers from the low and high context tables to
451 * temporary lists under the lock. This prevents the ISR from
452 * finding them while we process them outside the lock.
453 */
454 list_splice_init(&adev->dm.irq_handler_list_low_tab[src],
455 &low_handlers);
456 list_splice_init(&adev->dm.irq_handler_list_high_tab[src],
457 &high_handlers);
458
459 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
460
461 /*
462 * Cancel all pending work for the low-context handlers
463 * outside the lock. cancel_work_sync() may sleep and waits
464 * until any running work completes, preventing UAF.
465 */
466 list_for_each_safe(entry, tmp, &low_handlers) {
467 handler = list_entry(entry,
468 struct amdgpu_dm_irq_handler_data,
469 list);
470 cancel_work_sync(&handler->work);
471 }
472
473 /*
474 * High-context handlers are executed synchronously within ISR
475 * context (see amdgpu_dm_irq_immediate_work()) and have no
476 * work_struct, so there is no pending work to cancel here.
477 * They will be freed along with low_handlers after the loop.
478 */
479 }
480
481 /* Deallocate all handlers. */
482 list_for_each_safe(entry, tmp, &low_handlers) {
483 handler = list_entry(entry,
484 struct amdgpu_dm_irq_handler_data,
485 list);
486 list_del(&handler->list);
487 kfree(handler);
488 }
489
490 list_for_each_safe(entry, tmp, &high_handlers) {
491 handler = list_entry(entry,
492 struct amdgpu_dm_irq_handler_data,
493 list);
494 list_del(&handler->list);
495 kfree(handler);
496 }
497
498 if (adev->dm.vmin_vmax_wq) {
499 destroy_workqueue(adev->dm.vmin_vmax_wq);
500 adev->dm.vmin_vmax_wq = NULL;
501 }
502
503 if (adev->dm.irq_wq) {
504 destroy_workqueue(adev->dm.irq_wq);
505 adev->dm.irq_wq = NULL;
506 }
507 }
508 EXPORT_IF_KUNIT(amdgpu_dm_irq_fini);
509
amdgpu_dm_irq_suspend(struct amdgpu_device * adev)510 void amdgpu_dm_irq_suspend(struct amdgpu_device *adev)
511 {
512 struct drm_device *dev = adev_to_drm(adev);
513 int src;
514 struct list_head *hnd_list_l;
515 unsigned long irq_table_flags;
516 struct list_head *entry, *tmp;
517 struct amdgpu_dm_irq_handler_data *handler;
518
519 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
520
521 DRM_DEBUG_KMS("DM_IRQ: suspend\n");
522
523 /**
524 * Disable HW interrupt for HPD and HPDRX only since FLIP and VBLANK
525 * will be disabled from manage_dm_interrupts on disable CRTC.
526 *
527 * Disable the HW interrupt first, then flush any pending work. Since
528 * the HW interrupt is disabled under the lock, no new IRQ can be
529 * generated after the disable completes. Any work already queued by an
530 * in-flight ISR will be flushed below.
531 */
532 for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6RX; src++) {
533 hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
534 amdgpu_dm_irq_set(adev, src, false);
535
536 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
537
538 if (!list_empty(hnd_list_l)) {
539 list_for_each_safe(entry, tmp, hnd_list_l) {
540 handler = list_entry(
541 entry,
542 struct amdgpu_dm_irq_handler_data,
543 list);
544 flush_work(&handler->work);
545 }
546 }
547 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
548 }
549
550 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
551
552 if (dev->mode_config.poll_enabled)
553 drm_kms_helper_poll_disable(dev);
554 }
555 EXPORT_IF_KUNIT(amdgpu_dm_irq_suspend);
556
amdgpu_dm_irq_resume_early(struct amdgpu_device * adev)557 void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev)
558 {
559 int src;
560 struct list_head *hnd_list_h, *hnd_list_l;
561 unsigned long irq_table_flags;
562
563 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
564
565 drm_dbg(adev_to_drm(adev), "DM_IRQ: early resume\n");
566
567 /* re-enable short pulse interrupts HW interrupt */
568 for (src = DC_IRQ_SOURCE_HPD1RX; src <= DC_IRQ_SOURCE_HPD6RX; src++) {
569 hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
570 hnd_list_h = &adev->dm.irq_handler_list_high_tab[src];
571 if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h))
572 amdgpu_dm_irq_set(adev, src, true);
573 }
574
575 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
576 }
577 EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_early);
578
amdgpu_dm_irq_resume_late(struct amdgpu_device * adev)579 void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev)
580 {
581 struct drm_device *dev = adev_to_drm(adev);
582 int src;
583 struct list_head *hnd_list_h, *hnd_list_l;
584 unsigned long irq_table_flags;
585
586 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
587
588 drm_dbg(adev_to_drm(adev), "DM_IRQ: resume\n");
589
590 /**
591 * Renable HW interrupt for HPD and only since FLIP and VBLANK
592 * will be enabled from manage_dm_interrupts on enable CRTC.
593 */
594 for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6; src++) {
595 hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
596 hnd_list_h = &adev->dm.irq_handler_list_high_tab[src];
597 if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h))
598 amdgpu_dm_irq_set(adev, src, true);
599 }
600
601 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
602
603 if (dev->mode_config.poll_enabled)
604 drm_kms_helper_poll_enable(dev);
605 }
606 EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_late);
607
608 /*
609 * amdgpu_dm_irq_schedule_work - schedule all work items registered for the
610 * "irq_source".
611 */
amdgpu_dm_irq_schedule_work(struct amdgpu_device * adev,enum dc_irq_source irq_source)612 STATIC_IFN_KUNIT void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
613 enum dc_irq_source irq_source)
614 {
615 struct list_head *handler_list = &adev->dm.irq_handler_list_low_tab[irq_source];
616 struct amdgpu_dm_irq_handler_data *handler_data;
617 bool work_queued = false;
618 unsigned long irq_table_flags;
619
620 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
621
622 if (list_empty(handler_list))
623 goto out_unlock;
624
625 list_for_each_entry(handler_data, handler_list, list) {
626 if (queue_work(adev->dm.irq_wq, &handler_data->work)) {
627 work_queued = true;
628 break;
629 }
630 }
631
632 if (!work_queued) {
633 struct amdgpu_dm_irq_handler_data *handler_data_add;
634 /*get the amdgpu_dm_irq_handler_data of first item pointed by handler_list*/
635 handler_data = container_of(handler_list->next, struct amdgpu_dm_irq_handler_data, list);
636
637 /*allocate a new amdgpu_dm_irq_handler_data*/
638 handler_data_add = kzalloc_obj(*handler_data, GFP_ATOMIC);
639 if (!handler_data_add) {
640 DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n");
641 goto out_unlock;
642 }
643
644 /*copy new amdgpu_dm_irq_handler_data members from handler_data*/
645 handler_data_add->handler = handler_data->handler;
646 handler_data_add->handler_arg = handler_data->handler_arg;
647 handler_data_add->dm = handler_data->dm;
648 handler_data_add->irq_source = irq_source;
649
650 list_add_tail(&handler_data_add->list, handler_list);
651
652 INIT_WORK(&handler_data_add->work, dm_irq_work_func);
653
654 if (queue_work(adev->dm.irq_wq, &handler_data_add->work))
655 DRM_DEBUG("Queued work for handling interrupt from "
656 "display for IRQ source %d\n",
657 irq_source);
658 else
659 DRM_ERROR("Failed to queue work for handling interrupt "
660 "from display for IRQ source %d\n",
661 irq_source);
662 }
663
664 out_unlock:
665 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
666 }
667 EXPORT_IF_KUNIT(amdgpu_dm_irq_schedule_work);
668
669 /*
670 * amdgpu_dm_irq_immediate_work
671 * Callback high irq work immediately, don't send to work queue
672 */
amdgpu_dm_irq_immediate_work(struct amdgpu_device * adev,enum dc_irq_source irq_source)673 STATIC_IFN_KUNIT void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
674 enum dc_irq_source irq_source)
675 {
676 struct amdgpu_dm_irq_handler_data *handler_data;
677 unsigned long irq_table_flags;
678
679 DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
680
681 list_for_each_entry(handler_data,
682 &adev->dm.irq_handler_list_high_tab[irq_source],
683 list) {
684 /* Call a subcomponent which registered for immediate
685 * interrupt notification
686 */
687 handler_data->handler(handler_data->handler_arg);
688 }
689
690 DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
691 }
692 EXPORT_IF_KUNIT(amdgpu_dm_irq_immediate_work);
693
amdgpu_dm_irq_set(struct amdgpu_device * adev,enum dc_irq_source src,bool enable)694 bool amdgpu_dm_irq_set(struct amdgpu_device *adev, enum dc_irq_source src,
695 bool enable)
696 {
697 guard(spinlock_irqsave)(&adev->dm.irq_reg_lock);
698
699 return dc_interrupt_set(adev->dm.dc, src, enable);
700 }
701 EXPORT_IF_KUNIT(amdgpu_dm_irq_set);
702
amdgpu_dm_irq_ack(struct amdgpu_device * adev,enum dc_irq_source src)703 void amdgpu_dm_irq_ack(struct amdgpu_device *adev, enum dc_irq_source src)
704 {
705 guard(spinlock_irqsave)(&adev->dm.irq_reg_lock);
706
707 dc_interrupt_ack(adev->dm.dc, src);
708 }
709 EXPORT_IF_KUNIT(amdgpu_dm_irq_ack);
710
711 /**
712 * amdgpu_dm_irq_handler - Generic DM IRQ handler
713 * @adev: amdgpu base driver device containing the DM device
714 * @source: Unused
715 * @entry: Data about the triggered interrupt
716 *
717 * Calls all registered high irq work immediately, and schedules work for low
718 * irq. The DM IRQ table is used to find the corresponding handlers.
719 */
amdgpu_dm_irq_handler(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)720 STATIC_IFN_KUNIT int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
721 struct amdgpu_irq_src *source,
722 struct amdgpu_iv_entry *entry)
723 {
724
725 enum dc_irq_source src =
726 dc_interrupt_to_irq_source(
727 adev->dm.dc,
728 entry->src_id,
729 entry->src_data[0]);
730
731 amdgpu_dm_irq_ack(adev, src);
732
733 /* Call high irq work immediately */
734 amdgpu_dm_irq_immediate_work(adev, src);
735 /*Schedule low_irq work */
736 amdgpu_dm_irq_schedule_work(adev, src);
737
738 return 0;
739 }
740 EXPORT_IF_KUNIT(amdgpu_dm_irq_handler);
741
amdgpu_dm_hpd_to_dal_irq_source(unsigned int type)742 STATIC_IFN_KUNIT enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type)
743 {
744 switch (type) {
745 case AMDGPU_HPD_1:
746 return DC_IRQ_SOURCE_HPD1;
747 case AMDGPU_HPD_2:
748 return DC_IRQ_SOURCE_HPD2;
749 case AMDGPU_HPD_3:
750 return DC_IRQ_SOURCE_HPD3;
751 case AMDGPU_HPD_4:
752 return DC_IRQ_SOURCE_HPD4;
753 case AMDGPU_HPD_5:
754 return DC_IRQ_SOURCE_HPD5;
755 case AMDGPU_HPD_6:
756 return DC_IRQ_SOURCE_HPD6;
757 default:
758 return DC_IRQ_SOURCE_INVALID;
759 }
760 }
761 EXPORT_IF_KUNIT(amdgpu_dm_hpd_to_dal_irq_source);
762
amdgpu_dm_set_hpd_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)763 STATIC_IFN_KUNIT int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev,
764 struct amdgpu_irq_src *source,
765 unsigned int type,
766 enum amdgpu_interrupt_state state)
767 {
768 enum dc_irq_source src = amdgpu_dm_hpd_to_dal_irq_source(type);
769 bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
770
771 amdgpu_dm_irq_set(adev, src, st);
772 return 0;
773 }
774 EXPORT_IF_KUNIT(amdgpu_dm_set_hpd_irq_state);
775
dm_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int crtc_id,enum amdgpu_interrupt_state state,const enum irq_type dal_irq_type,const char * func)776 static inline int dm_irq_state(struct amdgpu_device *adev,
777 struct amdgpu_irq_src *source,
778 unsigned int crtc_id,
779 enum amdgpu_interrupt_state state,
780 const enum irq_type dal_irq_type,
781 const char *func)
782 {
783 bool st;
784 enum dc_irq_source irq_source;
785 struct dc *dc = adev->dm.dc;
786 struct amdgpu_crtc *acrtc = adev->mode_info.crtcs[crtc_id];
787
788 if (!acrtc) {
789 DRM_ERROR(
790 "%s: crtc is NULL at id :%d\n",
791 func,
792 crtc_id);
793 return 0;
794 }
795
796 if (acrtc->otg_inst == -1)
797 return 0;
798
799 irq_source = dal_irq_type + acrtc->otg_inst;
800
801 st = (state == AMDGPU_IRQ_STATE_ENABLE);
802
803 if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed)
804 dc_allow_idle_optimizations(dc, false);
805
806 amdgpu_dm_irq_set(adev, irq_source, st);
807 return 0;
808 }
809
amdgpu_dm_set_pflip_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int crtc_id,enum amdgpu_interrupt_state state)810 STATIC_IFN_KUNIT int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev,
811 struct amdgpu_irq_src *source,
812 unsigned int crtc_id,
813 enum amdgpu_interrupt_state state)
814 {
815 return dm_irq_state(
816 adev,
817 source,
818 crtc_id,
819 state,
820 IRQ_TYPE_PFLIP,
821 __func__);
822 }
823 EXPORT_IF_KUNIT(amdgpu_dm_set_pflip_irq_state);
824
amdgpu_dm_set_crtc_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int crtc_id,enum amdgpu_interrupt_state state)825 STATIC_IFN_KUNIT int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
826 struct amdgpu_irq_src *source,
827 unsigned int crtc_id,
828 enum amdgpu_interrupt_state state)
829 {
830 return dm_irq_state(
831 adev,
832 source,
833 crtc_id,
834 state,
835 IRQ_TYPE_VBLANK,
836 __func__);
837 }
838 EXPORT_IF_KUNIT(amdgpu_dm_set_crtc_irq_state);
839
amdgpu_dm_set_vline0_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int crtc_id,enum amdgpu_interrupt_state state)840 STATIC_IFN_KUNIT int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev,
841 struct amdgpu_irq_src *source,
842 unsigned int crtc_id,
843 enum amdgpu_interrupt_state state)
844 {
845 return dm_irq_state(
846 adev,
847 source,
848 crtc_id,
849 state,
850 IRQ_TYPE_VLINE0,
851 __func__);
852 }
853 EXPORT_IF_KUNIT(amdgpu_dm_set_vline0_irq_state);
854
amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int crtc_id,enum amdgpu_interrupt_state state)855 STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
856 struct amdgpu_irq_src *source,
857 unsigned int crtc_id,
858 enum amdgpu_interrupt_state state)
859 {
860 enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX;
861 bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
862
863 amdgpu_dm_irq_set(adev, irq_source, st);
864 return 0;
865 }
866 EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_outbox_irq_state);
867
amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int crtc_id,enum amdgpu_interrupt_state state)868 STATIC_IFN_KUNIT int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
869 struct amdgpu_irq_src *source,
870 unsigned int crtc_id,
871 enum amdgpu_interrupt_state state)
872 {
873 return dm_irq_state(
874 adev,
875 source,
876 crtc_id,
877 state,
878 IRQ_TYPE_VUPDATE,
879 __func__);
880 }
881 EXPORT_IF_KUNIT(amdgpu_dm_set_vupdate_irq_state);
882
amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)883 STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev,
884 struct amdgpu_irq_src *source,
885 unsigned int type,
886 enum amdgpu_interrupt_state state)
887 {
888 enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0;
889 bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
890
891 amdgpu_dm_irq_set(adev, irq_source, st);
892 return 0;
893 }
894 EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_trace_irq_state);
895
896 static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = {
897 .set = amdgpu_dm_set_crtc_irq_state,
898 .process = amdgpu_dm_irq_handler,
899 };
900
901 static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = {
902 .set = amdgpu_dm_set_vline0_irq_state,
903 .process = amdgpu_dm_irq_handler,
904 };
905
906 static const struct amdgpu_irq_src_funcs dm_dmub_outbox_irq_funcs = {
907 .set = amdgpu_dm_set_dmub_outbox_irq_state,
908 .process = amdgpu_dm_irq_handler,
909 };
910
911 static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = {
912 .set = amdgpu_dm_set_vupdate_irq_state,
913 .process = amdgpu_dm_irq_handler,
914 };
915
916 static const struct amdgpu_irq_src_funcs dm_dmub_trace_irq_funcs = {
917 .set = amdgpu_dm_set_dmub_trace_irq_state,
918 .process = amdgpu_dm_irq_handler,
919 };
920
921 static const struct amdgpu_irq_src_funcs dm_pageflip_irq_funcs = {
922 .set = amdgpu_dm_set_pflip_irq_state,
923 .process = amdgpu_dm_irq_handler,
924 };
925
926 static const struct amdgpu_irq_src_funcs dm_hpd_irq_funcs = {
927 .set = amdgpu_dm_set_hpd_irq_state,
928 .process = amdgpu_dm_irq_handler,
929 };
930
amdgpu_dm_set_irq_funcs(struct amdgpu_device * adev)931 void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
932 {
933 adev->crtc_irq.num_types = adev->mode_info.num_crtc;
934 adev->crtc_irq.funcs = &dm_crtc_irq_funcs;
935
936 adev->vline0_irq.num_types = adev->mode_info.num_crtc;
937 adev->vline0_irq.funcs = &dm_vline0_irq_funcs;
938
939 adev->dmub_outbox_irq.num_types = 1;
940 adev->dmub_outbox_irq.funcs = &dm_dmub_outbox_irq_funcs;
941
942 adev->vupdate_irq.num_types = adev->mode_info.num_crtc;
943 adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs;
944
945 adev->dmub_trace_irq.num_types = 1;
946 adev->dmub_trace_irq.funcs = &dm_dmub_trace_irq_funcs;
947
948 adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
949 adev->pageflip_irq.funcs = &dm_pageflip_irq_funcs;
950
951 adev->hpd_irq.num_types = adev->mode_info.num_hpd;
952 adev->hpd_irq.funcs = &dm_hpd_irq_funcs;
953 }
954 EXPORT_IF_KUNIT(amdgpu_dm_set_irq_funcs);
955
amdgpu_dm_outbox_init(struct amdgpu_device * adev)956 void amdgpu_dm_outbox_init(struct amdgpu_device *adev)
957 {
958 amdgpu_dm_irq_set(adev, DC_IRQ_SOURCE_DMCUB_OUTBOX, true);
959 }
960 EXPORT_IF_KUNIT(amdgpu_dm_outbox_init);
961
962 /**
963 * amdgpu_dm_hpd_init - hpd setup callback.
964 *
965 * @adev: amdgpu_device pointer
966 *
967 * Setup the hpd pins used by the card (evergreen+).
968 * Enable the pin, set the polarity, and enable the hpd interrupts.
969 */
amdgpu_dm_hpd_init(struct amdgpu_device * adev)970 void amdgpu_dm_hpd_init(struct amdgpu_device *adev)
971 {
972 struct drm_device *dev = adev_to_drm(adev);
973 struct drm_connector *connector;
974 struct drm_connector_list_iter iter;
975 int irq_type;
976 int i;
977 bool use_polling = false;
978
979 /* First, clear all hpd and hpdrx interrupts */
980 for (i = DC_IRQ_SOURCE_HPD1; i <= DC_IRQ_SOURCE_HPD6RX; i++) {
981 if (!amdgpu_dm_irq_set(adev, i, false))
982 drm_err(dev, "Failed to clear hpd(rx) source=%d on init\n",
983 i);
984 }
985
986 drm_connector_list_iter_begin(dev, &iter);
987 drm_for_each_connector_iter(connector, &iter) {
988 struct amdgpu_dm_connector *amdgpu_dm_connector;
989 const struct dc_link *dc_link;
990
991 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
992 continue;
993
994 amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
995 dc_link = amdgpu_dm_connector->dc_link;
996 if (!dc_link)
997 continue;
998
999 /*
1000 * Analog connectors may be hot-plugged unlike other connector
1001 * types that don't support HPD. Only poll analog connectors.
1002 */
1003 use_polling |= dc_connector_supports_analog(dc_link->link_id.id);
1004
1005 /*
1006 * Get a base driver irq reference for hpd ints for the lifetime
1007 * of dm. Note that only hpd interrupt types are registered with
1008 * base driver; hpd_rx types aren't. IOW, amdgpu_irq_get/put on
1009 * hpd_rx isn't available. DM currently controls hpd_rx
1010 * explicitly with amdgpu_dm_irq_set()
1011 */
1012 if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
1013 irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1;
1014 /*
1015 * TODO: There's a mismatch between mode_info.num_hpd
1016 * and what bios reports as the # of connectors with hpd
1017 * sources. Since the # of hpd source types registered
1018 * with base driver == mode_info.num_hpd, we have to
1019 * fallback to amdgpu_dm_irq_set for the remaining types.
1020 */
1021 if (irq_type < adev->mode_info.num_hpd) {
1022 if (amdgpu_irq_get(adev, &adev->hpd_irq, irq_type))
1023 drm_err(dev, "DM_IRQ: Failed get HPD for source=%d)!\n",
1024 dc_link->irq_source_hpd);
1025 } else {
1026 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd,
1027 true);
1028 }
1029 }
1030
1031 if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
1032 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx,
1033 true);
1034 }
1035 }
1036 drm_connector_list_iter_end(&iter);
1037
1038 if (use_polling)
1039 drm_kms_helper_poll_init(dev);
1040 }
1041 EXPORT_IF_KUNIT(amdgpu_dm_hpd_init);
1042
1043 /**
1044 * amdgpu_dm_hpd_fini - hpd tear down callback.
1045 *
1046 * @adev: amdgpu_device pointer
1047 *
1048 * Tear down the hpd pins used by the card (evergreen+).
1049 * Disable the hpd interrupts.
1050 */
amdgpu_dm_hpd_fini(struct amdgpu_device * adev)1051 void amdgpu_dm_hpd_fini(struct amdgpu_device *adev)
1052 {
1053 struct drm_device *dev = adev_to_drm(adev);
1054 struct drm_connector *connector;
1055 struct drm_connector_list_iter iter;
1056 int irq_type;
1057
1058 drm_connector_list_iter_begin(dev, &iter);
1059 drm_for_each_connector_iter(connector, &iter) {
1060 struct amdgpu_dm_connector *amdgpu_dm_connector;
1061 const struct dc_link *dc_link;
1062
1063 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
1064 continue;
1065
1066 amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
1067 dc_link = amdgpu_dm_connector->dc_link;
1068
1069 if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
1070 irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1;
1071
1072 /* TODO: See same TODO in amdgpu_dm_hpd_init() */
1073 if (irq_type < adev->mode_info.num_hpd) {
1074 if (amdgpu_irq_put(adev, &adev->hpd_irq, irq_type))
1075 drm_err(dev, "DM_IRQ: Failed put HPD for source=%d!\n",
1076 dc_link->irq_source_hpd);
1077 } else {
1078 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd,
1079 false);
1080 }
1081 }
1082
1083 if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
1084 amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx,
1085 false);
1086 }
1087 }
1088 drm_connector_list_iter_end(&iter);
1089
1090 if (dev->mode_config.poll_enabled)
1091 drm_kms_helper_poll_fini(dev);
1092 }
1093 EXPORT_IF_KUNIT(amdgpu_dm_hpd_fini);
1094
1095 /* ========== HPD handling ========== */
force_connector_state(struct amdgpu_dm_connector * aconnector,enum drm_connector_force force_state)1096 static void force_connector_state(
1097 struct amdgpu_dm_connector *aconnector,
1098 enum drm_connector_force force_state)
1099 {
1100 struct drm_connector *connector = &aconnector->base;
1101
1102 mutex_lock(&connector->dev->mode_config.mutex);
1103 aconnector->base.force = force_state;
1104 mutex_unlock(&connector->dev->mode_config.mutex);
1105
1106 mutex_lock(&aconnector->hpd_lock);
1107 drm_kms_helper_connector_hotplug_event(connector);
1108 mutex_unlock(&aconnector->hpd_lock);
1109 }
1110
dm_handle_hpd_rx_offload_work(struct work_struct * work)1111 STATIC_IFN_KUNIT void dm_handle_hpd_rx_offload_work(struct work_struct *work)
1112 {
1113 struct hpd_rx_irq_offload_work *offload_work;
1114 struct amdgpu_dm_connector *aconnector;
1115 struct dc_link *dc_link;
1116 struct amdgpu_device *adev;
1117 enum dc_connection_type new_connection_type = dc_connection_none;
1118 unsigned long flags;
1119 union test_response test_response;
1120
1121 memset(&test_response, 0, sizeof(test_response));
1122
1123 offload_work = container_of(work, struct hpd_rx_irq_offload_work, work);
1124 aconnector = offload_work->offload_wq->aconnector;
1125 adev = offload_work->adev;
1126
1127 if (!aconnector) {
1128 drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work");
1129 goto skip;
1130 }
1131
1132 dc_link = aconnector->dc_link;
1133
1134 mutex_lock(&aconnector->hpd_lock);
1135 if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
1136 drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
1137 mutex_unlock(&aconnector->hpd_lock);
1138
1139 if (new_connection_type == dc_connection_none)
1140 goto skip;
1141
1142 if (amdgpu_in_reset(adev))
1143 goto skip;
1144
1145 if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY ||
1146 offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
1147 dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT);
1148 spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
1149 offload_work->offload_wq->is_handling_mst_msg_rdy_event = false;
1150 spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
1151 goto skip;
1152 }
1153
1154 mutex_lock(&adev->dm.dc_lock);
1155 if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
1156 dc_link_dp_handle_automated_test(dc_link);
1157
1158 if (aconnector->timing_changed) {
1159 /* force connector disconnect and reconnect */
1160 force_connector_state(aconnector, DRM_FORCE_OFF);
1161 msleep(100);
1162 force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED);
1163 }
1164
1165 test_response.bits.ACK = 1;
1166
1167 core_link_write_dpcd(
1168 dc_link,
1169 DP_TEST_RESPONSE,
1170 &test_response.raw,
1171 sizeof(test_response));
1172 } else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) &&
1173 dc_link_check_link_loss_status(dc_link, &offload_work->data) &&
1174 dc_link_dp_allow_hpd_rx_irq(dc_link)) {
1175 /* offload_work->data is from handle_hpd_rx_irq->
1176 * schedule_hpd_rx_offload_work.this is defer handle
1177 * for hpd short pulse. upon here, link status may be
1178 * changed, need get latest link status from dpcd
1179 * registers. if link status is good, skip run link
1180 * training again.
1181 */
1182 union hpd_irq_data irq_data;
1183
1184 memset(&irq_data, 0, sizeof(irq_data));
1185
1186 /* before dc_link_dp_handle_link_loss, allow new link lost handle
1187 * request be added to work queue if link lost at end of dc_link_
1188 * dp_handle_link_loss
1189 */
1190 spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
1191 offload_work->offload_wq->is_handling_link_loss = false;
1192 spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
1193
1194 if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) &&
1195 dc_link_check_link_loss_status(dc_link, &irq_data))
1196 dc_link_dp_handle_link_loss(dc_link);
1197 }
1198 mutex_unlock(&adev->dm.dc_lock);
1199
1200 skip:
1201 kfree(offload_work);
1202
1203 }
1204 EXPORT_IF_KUNIT(dm_handle_hpd_rx_offload_work);
1205
amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device * adev)1206 struct hpd_rx_irq_offload_work_queue *amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device *adev)
1207 {
1208 struct dc *dc = adev->dm.dc;
1209 int max_caps = dc->caps.max_links;
1210 int i = 0;
1211 struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL;
1212
1213 hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps);
1214
1215 if (!hpd_rx_offload_wq)
1216 return NULL;
1217
1218
1219 for (i = 0; i < max_caps; i++) {
1220 hpd_rx_offload_wq[i].wq =
1221 create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq");
1222
1223 if (hpd_rx_offload_wq[i].wq == NULL) {
1224 drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!");
1225 goto out_err;
1226 }
1227
1228 spin_lock_init(&hpd_rx_offload_wq[i].offload_lock);
1229 }
1230
1231 return hpd_rx_offload_wq;
1232
1233 out_err:
1234 for (i = 0; i < max_caps; i++) {
1235 if (hpd_rx_offload_wq[i].wq)
1236 destroy_workqueue(hpd_rx_offload_wq[i].wq);
1237 }
1238 kfree(hpd_rx_offload_wq);
1239 return NULL;
1240 }
1241 EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_create_workqueue);
1242
amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager * dm)1243 void amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm)
1244 {
1245 int i;
1246
1247 if (dm->hpd_rx_offload_wq) {
1248 for (i = 0; i < dm->dc->caps.max_links; i++)
1249 flush_workqueue(dm->hpd_rx_offload_wq[i].wq);
1250 }
1251 }
1252 EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_work_suspend);
1253
are_sinks_equal(const struct dc_sink * sink1,const struct dc_sink * sink2)1254 STATIC_IFN_KUNIT bool are_sinks_equal(const struct dc_sink *sink1, const struct dc_sink *sink2)
1255 {
1256 if (!sink1 || !sink2)
1257 return false;
1258 if (sink1->sink_signal != sink2->sink_signal)
1259 return false;
1260
1261 if (sink1->dc_edid.length != sink2->dc_edid.length)
1262 return false;
1263
1264 if (memcmp(sink1->dc_edid.raw_edid, sink2->dc_edid.raw_edid,
1265 sink1->dc_edid.length) != 0)
1266 return false;
1267 return true;
1268 }
1269 EXPORT_IF_KUNIT(are_sinks_equal);
1270
1271
1272 /**
1273 * DOC: amdgpu_dm_hdmi_hpd_debounce_work
1274 *
1275 * HDMI HPD debounce delay in milliseconds. When an HDMI display toggles HPD
1276 * (such as during power save transitions), this delay determines how long to
1277 * wait before processing the HPD event. This allows distinguishing between a
1278 * physical unplug (>hdmi_hpd_debounce_delay)
1279 * and a spontaneous RX HPD toggle (<hdmi_hpd_debounce_delay).
1280 *
1281 * If the toggle is less than this delay, the driver compares sink capabilities
1282 * and permits a hotplug event if they changed.
1283 *
1284 * The default value of 1500ms was chosen based on experimental testing with
1285 * various monitors that exhibit spontaneous HPD toggling behavior.
1286 */
amdgpu_dm_hdmi_hpd_debounce_work(struct work_struct * work)1287 void amdgpu_dm_hdmi_hpd_debounce_work(struct work_struct *work)
1288 {
1289 struct amdgpu_dm_connector *aconnector =
1290 container_of(to_delayed_work(work), struct amdgpu_dm_connector,
1291 hdmi_hpd_debounce_work);
1292 struct drm_connector *connector = &aconnector->base;
1293 struct drm_device *dev = connector->dev;
1294 struct amdgpu_device *adev = drm_to_adev(dev);
1295 struct dc *dc = aconnector->dc_link->ctx->dc;
1296 bool fake_reconnect = false;
1297 bool reallow_idle = false;
1298 bool ret = false;
1299
1300 guard(mutex)(&aconnector->hpd_lock);
1301
1302 /* Re-detect the display */
1303 scoped_guard(mutex, &adev->dm.dc_lock) {
1304 if (dc->caps.ips_support && dc->ctx->dmub_srv->idle_allowed) {
1305 dc_allow_idle_optimizations(dc, false);
1306 reallow_idle = true;
1307 }
1308 ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD);
1309 }
1310
1311 if (ret) {
1312 /* Apply workaround delay for certain panels */
1313 amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
1314 /* Compare sinks to determine if this was a spontaneous HPD toggle */
1315 if (are_sinks_equal(aconnector->dc_link->local_sink, aconnector->hdmi_prev_sink)) {
1316 /*
1317 * Sinks match - this was a spontaneous HDMI HPD toggle.
1318 */
1319 drm_dbg_kms(dev, "HDMI HPD: Sink unchanged after debounce, internal re-enable\n");
1320 fake_reconnect = true;
1321 }
1322
1323 /* Update connector state */
1324 amdgpu_dm_update_connector_after_detect(aconnector);
1325
1326 drm_modeset_lock_all(dev);
1327 dm_restore_drm_connector_state(dev, connector);
1328 drm_modeset_unlock_all(dev);
1329
1330 /* Only notify OS if sink actually changed */
1331 if (!fake_reconnect && aconnector->base.force == DRM_FORCE_UNSPECIFIED)
1332 drm_kms_helper_hotplug_event(dev);
1333 }
1334
1335 /* Release the cached sink reference */
1336 if (aconnector->hdmi_prev_sink) {
1337 dc_sink_release(aconnector->hdmi_prev_sink);
1338 aconnector->hdmi_prev_sink = NULL;
1339 }
1340
1341 scoped_guard(mutex, &adev->dm.dc_lock) {
1342 if (reallow_idle && dc->caps.ips_support)
1343 dc_allow_idle_optimizations(dc, true);
1344 }
1345 }
1346 EXPORT_IF_KUNIT(amdgpu_dm_hdmi_hpd_debounce_work);
1347
handle_hpd_irq_helper(struct amdgpu_dm_connector * aconnector,enum dc_detect_reason reason)1348 STATIC_IFN_KUNIT void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector,
1349 enum dc_detect_reason reason)
1350 {
1351 struct drm_connector *connector = &aconnector->base;
1352 struct drm_device *dev = connector->dev;
1353 enum dc_connection_type new_connection_type = dc_connection_none;
1354 struct amdgpu_device *adev = drm_to_adev(dev);
1355 struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state);
1356 struct dc *dc = aconnector->dc_link->ctx->dc;
1357 bool ret = false;
1358 bool debounce_required = false;
1359
1360 if (adev->dm.disable_hpd_irq)
1361 return;
1362
1363 /*
1364 * In case of failure or MST no need to update connector status or notify the OS
1365 * since (for MST case) MST does this in its own context.
1366 */
1367 guard(mutex)(&aconnector->hpd_lock);
1368
1369 if (adev->dm.hdcp_workqueue) {
1370 hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index);
1371 dm_con_state->update_hdcp = true;
1372 }
1373 if (aconnector->fake_enable)
1374 aconnector->fake_enable = false;
1375
1376 aconnector->timing_changed = false;
1377
1378 if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type))
1379 drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
1380
1381 /*
1382 * Check for HDMI disconnect with debounce enabled.
1383 */
1384 debounce_required = (aconnector->hdmi_hpd_debounce_delay_ms > 0 &&
1385 dc_is_hdmi_signal(aconnector->dc_link->connector_signal) &&
1386 new_connection_type == dc_connection_none &&
1387 aconnector->dc_link->local_sink != NULL);
1388
1389 if (aconnector->base.force && new_connection_type == dc_connection_none) {
1390 amdgpu_dm_emulated_link_detect(aconnector->dc_link);
1391
1392 drm_modeset_lock_all(dev);
1393 dm_restore_drm_connector_state(dev, connector);
1394 drm_modeset_unlock_all(dev);
1395
1396 if (aconnector->base.force == DRM_FORCE_UNSPECIFIED ||
1397 reason == DETECT_REASON_HPDRX)
1398 drm_kms_helper_connector_hotplug_event(connector);
1399 } else if (debounce_required) {
1400 /*
1401 * HDMI disconnect detected - schedule delayed work instead of
1402 * processing immediately. This allows us to coalesce spurious
1403 * HDMI signals from physical unplugs.
1404 */
1405 drm_dbg_kms(dev, "HDMI HPD: Disconnect detected, scheduling debounce work (%u ms)\n",
1406 aconnector->hdmi_hpd_debounce_delay_ms);
1407
1408 /* Cache the current sink for later comparison */
1409 if (aconnector->hdmi_prev_sink)
1410 dc_sink_release(aconnector->hdmi_prev_sink);
1411 aconnector->hdmi_prev_sink = aconnector->dc_link->local_sink;
1412 if (aconnector->hdmi_prev_sink)
1413 dc_sink_retain(aconnector->hdmi_prev_sink);
1414
1415 /* Schedule delayed detection. */
1416 if (mod_delayed_work(system_percpu_wq,
1417 &aconnector->hdmi_hpd_debounce_work,
1418 msecs_to_jiffies(aconnector->hdmi_hpd_debounce_delay_ms)))
1419 drm_dbg_kms(dev, "HDMI HPD: Re-scheduled debounce work\n");
1420
1421 } else {
1422
1423 /* If the aconnector->hdmi_hpd_debounce_work is scheduled, exit early */
1424 if (delayed_work_pending(&aconnector->hdmi_hpd_debounce_work))
1425 return;
1426
1427 scoped_guard(mutex, &adev->dm.dc_lock) {
1428 dc_exit_ips_for_hw_access(dc);
1429 ret = dc_link_detect(aconnector->dc_link, reason);
1430 }
1431 if (ret) {
1432 /* w/a delay for certain panels */
1433 amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
1434 amdgpu_dm_update_connector_after_detect(aconnector);
1435
1436 drm_modeset_lock_all(dev);
1437 dm_restore_drm_connector_state(dev, connector);
1438 drm_modeset_unlock_all(dev);
1439
1440 if (aconnector->base.force == DRM_FORCE_UNSPECIFIED ||
1441 reason == DETECT_REASON_HPDRX)
1442 drm_kms_helper_connector_hotplug_event(connector);
1443 }
1444 }
1445 }
1446 EXPORT_IF_KUNIT(handle_hpd_irq_helper);
1447
handle_hpd_irq(void * param)1448 STATIC_IFN_KUNIT void handle_hpd_irq(void *param)
1449 {
1450 struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param;
1451
1452 handle_hpd_irq_helper(aconnector, DETECT_REASON_HPD);
1453
1454 }
1455 EXPORT_IF_KUNIT(handle_hpd_irq);
1456
schedule_hpd_rx_offload_work(struct amdgpu_device * adev,struct hpd_rx_irq_offload_work_queue * offload_wq,union hpd_irq_data hpd_irq_data)1457 STATIC_IFN_KUNIT void schedule_hpd_rx_offload_work(struct amdgpu_device *adev, struct hpd_rx_irq_offload_work_queue *offload_wq,
1458 union hpd_irq_data hpd_irq_data)
1459 {
1460 struct hpd_rx_irq_offload_work *offload_work = kzalloc_obj(*offload_work);
1461
1462 if (!offload_work) {
1463 drm_err(adev_to_drm(adev), "Failed to allocate hpd_rx_irq_offload_work.\n");
1464 return;
1465 }
1466
1467 INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
1468 offload_work->data = hpd_irq_data;
1469 offload_work->offload_wq = offload_wq;
1470 offload_work->adev = adev;
1471
1472 queue_work(offload_wq->wq, &offload_work->work);
1473 drm_dbg_kms(adev_to_drm(adev), "queue work to handle hpd_rx offload work");
1474 }
1475 EXPORT_IF_KUNIT(schedule_hpd_rx_offload_work);
1476
handle_hpd_rx_irq(void * param)1477 STATIC_IFN_KUNIT void handle_hpd_rx_irq(void *param)
1478 {
1479 struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param;
1480 struct drm_connector *connector = &aconnector->base;
1481 struct drm_device *dev = connector->dev;
1482 struct dc_link *dc_link = aconnector->dc_link;
1483 bool is_mst_root_connector = aconnector->mst_mgr.mst_state;
1484 bool result = false;
1485 enum dc_connection_type new_connection_type = dc_connection_none;
1486 struct amdgpu_device *adev = drm_to_adev(dev);
1487 union hpd_irq_data hpd_irq_data;
1488 bool link_loss = false;
1489 bool has_left_work = false;
1490 int idx = dc_link->link_index;
1491 struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx];
1492 struct dc *dc = aconnector->dc_link->ctx->dc;
1493
1494 memset(&hpd_irq_data, 0, sizeof(hpd_irq_data));
1495
1496 if (adev->dm.disable_hpd_irq)
1497 return;
1498
1499 /*
1500 * TODO:Temporary add mutex to protect hpd interrupt not have a gpio
1501 * conflict, after implement i2c helper, this mutex should be
1502 * retired.
1503 */
1504 mutex_lock(&aconnector->hpd_lock);
1505
1506 result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data,
1507 &link_loss, true, &has_left_work);
1508
1509 if (!has_left_work)
1510 goto out;
1511
1512 if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
1513 schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
1514 goto out;
1515 }
1516
1517 if (dc_link_dp_allow_hpd_rx_irq(dc_link)) {
1518 if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY ||
1519 hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
1520 bool skip = false;
1521
1522 /*
1523 * DOWN_REP_MSG_RDY is also handled by polling method
1524 * mgr->cbs->poll_hpd_irq()
1525 */
1526 spin_lock(&offload_wq->offload_lock);
1527 skip = offload_wq->is_handling_mst_msg_rdy_event;
1528
1529 if (!skip)
1530 offload_wq->is_handling_mst_msg_rdy_event = true;
1531
1532 spin_unlock(&offload_wq->offload_lock);
1533
1534 if (!skip)
1535 schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
1536
1537 goto out;
1538 }
1539
1540 if (link_loss) {
1541 bool skip = false;
1542
1543 spin_lock(&offload_wq->offload_lock);
1544 skip = offload_wq->is_handling_link_loss;
1545
1546 if (!skip)
1547 offload_wq->is_handling_link_loss = true;
1548
1549 spin_unlock(&offload_wq->offload_lock);
1550
1551 if (!skip)
1552 schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
1553
1554 goto out;
1555 }
1556 }
1557
1558 out:
1559 if (result && !is_mst_root_connector) {
1560 /* Downstream Port status changed. */
1561 if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
1562 drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
1563
1564 if (aconnector->base.force && new_connection_type == dc_connection_none) {
1565 amdgpu_dm_emulated_link_detect(dc_link);
1566
1567 if (aconnector->fake_enable)
1568 aconnector->fake_enable = false;
1569
1570 amdgpu_dm_update_connector_after_detect(aconnector);
1571
1572
1573 drm_modeset_lock_all(dev);
1574 dm_restore_drm_connector_state(dev, connector);
1575 drm_modeset_unlock_all(dev);
1576
1577 drm_kms_helper_connector_hotplug_event(connector);
1578 } else {
1579 bool ret = false;
1580
1581 mutex_lock(&adev->dm.dc_lock);
1582 dc_exit_ips_for_hw_access(dc);
1583 ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX);
1584 mutex_unlock(&adev->dm.dc_lock);
1585
1586 if (ret) {
1587 if (aconnector->fake_enable)
1588 aconnector->fake_enable = false;
1589
1590 amdgpu_dm_update_connector_after_detect(aconnector);
1591
1592 drm_modeset_lock_all(dev);
1593 dm_restore_drm_connector_state(dev, connector);
1594 drm_modeset_unlock_all(dev);
1595
1596 drm_kms_helper_connector_hotplug_event(connector);
1597 }
1598 }
1599 }
1600 if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) {
1601 if (adev->dm.hdcp_workqueue)
1602 hdcp_handle_cpirq(adev->dm.hdcp_workqueue, aconnector->base.index);
1603 }
1604
1605 if (dc_link->type != dc_connection_mst_branch)
1606 drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
1607
1608 mutex_unlock(&aconnector->hpd_lock);
1609 }
1610 EXPORT_IF_KUNIT(handle_hpd_rx_irq);
1611
1612 /**
1613 * dmub_hpd_callback - DMUB HPD interrupt processing callback.
1614 * @adev: amdgpu_device pointer
1615 * @notify: dmub notification structure
1616 *
1617 * Dmub Hpd interrupt processing callback. Gets displayindex through the
1618 * ink index and calls helper to do the processing.
1619 */
dmub_hpd_callback(struct amdgpu_device * adev,struct dmub_notification * notify)1620 STATIC_IFN_KUNIT void dmub_hpd_callback(struct amdgpu_device *adev,
1621 struct dmub_notification *notify)
1622 {
1623 struct amdgpu_dm_connector *aconnector;
1624 struct amdgpu_dm_connector *hpd_aconnector = NULL;
1625 struct drm_connector *connector;
1626 struct drm_connector_list_iter iter;
1627 struct dc_link *link;
1628 u8 link_index = 0;
1629 struct drm_device *dev;
1630
1631 if (adev == NULL)
1632 return;
1633
1634 if (notify == NULL) {
1635 drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL");
1636 return;
1637 }
1638
1639 if (notify->link_index > adev->dm.dc->link_count) {
1640 drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index);
1641 return;
1642 }
1643
1644 /* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */
1645 if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) {
1646 drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n");
1647 return;
1648 }
1649
1650 link_index = notify->link_index;
1651 link = adev->dm.dc->links[link_index];
1652 dev = adev->dm.ddev;
1653
1654 drm_connector_list_iter_begin(dev, &iter);
1655 drm_for_each_connector_iter(connector, &iter) {
1656
1657 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
1658 continue;
1659
1660 aconnector = to_amdgpu_dm_connector(connector);
1661 if (link && aconnector->dc_link == link) {
1662 if (notify->type == DMUB_NOTIFICATION_HPD)
1663 drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index);
1664 else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ)
1665 drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index);
1666 else
1667 drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n",
1668 notify->type, link_index);
1669
1670 hpd_aconnector = aconnector;
1671 break;
1672 }
1673 }
1674 drm_connector_list_iter_end(&iter);
1675
1676 if (hpd_aconnector) {
1677 if (notify->type == DMUB_NOTIFICATION_HPD) {
1678 if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG))
1679 drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index);
1680 handle_hpd_irq_helper(hpd_aconnector, DETECT_REASON_HPD);
1681 } else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) {
1682 handle_hpd_rx_irq(hpd_aconnector);
1683 }
1684 }
1685 }
1686 EXPORT_IF_KUNIT(dmub_hpd_callback);
1687
1688 /**
1689 * dmub_hpd_sense_callback - DMUB HPD sense processing callback.
1690 * @adev: amdgpu_device pointer
1691 * @notify: dmub notification structure
1692 *
1693 * HPD sense changes can occur during low power states and need to be
1694 * notified from firmware to driver.
1695 */
dmub_hpd_sense_callback(struct amdgpu_device * adev,struct dmub_notification * notify)1696 STATIC_IFN_KUNIT void dmub_hpd_sense_callback(struct amdgpu_device *adev,
1697 struct dmub_notification *notify)
1698 {
1699 drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n");
1700 }
1701 EXPORT_IF_KUNIT(dmub_hpd_sense_callback);
1702
amdgpu_dm_register_hpd_handlers(struct amdgpu_device * adev)1703 int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev)
1704 {
1705 struct drm_device *dev = adev_to_drm(adev);
1706 struct drm_connector *connector;
1707 struct amdgpu_dm_connector *aconnector;
1708 const struct dc_link *dc_link;
1709 struct dc_interrupt_params int_params = {0};
1710
1711 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
1712 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
1713
1714 if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
1715 if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD,
1716 dmub_hpd_callback, true)) {
1717 drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
1718 return -EINVAL;
1719 }
1720
1721 if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ,
1722 dmub_hpd_callback, true)) {
1723 drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
1724 return -EINVAL;
1725 }
1726
1727 if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_SENSE_NOTIFY,
1728 dmub_hpd_sense_callback, true)) {
1729 drm_err(adev_to_drm(adev), "fail to register dmub hpd sense callback");
1730 return -EINVAL;
1731 }
1732 }
1733
1734 list_for_each_entry(connector,
1735 &dev->mode_config.connector_list, head) {
1736
1737 if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
1738 continue;
1739
1740 aconnector = to_amdgpu_dm_connector(connector);
1741 dc_link = aconnector->dc_link;
1742
1743 if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
1744 int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
1745 int_params.irq_source = dc_link->irq_source_hpd;
1746
1747 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
1748 int_params.irq_source < DC_IRQ_SOURCE_HPD1 ||
1749 int_params.irq_source > DC_IRQ_SOURCE_HPD6) {
1750 drm_err(adev_to_drm(adev),
1751 "Failed to register hpd irq %u for %s!\n",
1752 int_params.irq_source, connector->name);
1753 return -EINVAL;
1754 }
1755
1756 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
1757 handle_hpd_irq, (void *) aconnector))
1758 return -ENOMEM;
1759 }
1760
1761 if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
1762
1763 /* Also register for DP short pulse (hpd_rx). */
1764 int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
1765 int_params.irq_source = dc_link->irq_source_hpd_rx;
1766
1767 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
1768 int_params.irq_source < DC_IRQ_SOURCE_HPD1RX ||
1769 int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) {
1770 drm_err(adev_to_drm(adev),
1771 "Failed to register hpd rx irq %u for %s!\n",
1772 int_params.irq_source, connector->name);
1773 return -EINVAL;
1774 }
1775
1776 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
1777 handle_hpd_rx_irq, (void *) aconnector))
1778 return -ENOMEM;
1779 }
1780 }
1781 return 0;
1782 }
1783 EXPORT_IF_KUNIT(amdgpu_dm_register_hpd_handlers);
1784
1785 /* ========== IRQ handlers ========== */
1786 struct amdgpu_crtc *
amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device * adev,int otg_inst)1787 amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device *adev,
1788 int otg_inst)
1789 {
1790 struct drm_device *dev = adev_to_drm(adev);
1791 struct drm_crtc *crtc;
1792 struct amdgpu_crtc *amdgpu_crtc;
1793
1794 if (WARN_ON(otg_inst == -1))
1795 return adev->mode_info.crtcs[0];
1796
1797 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1798 amdgpu_crtc = to_amdgpu_crtc(crtc);
1799
1800 if (amdgpu_crtc->otg_inst == otg_inst)
1801 return amdgpu_crtc;
1802 }
1803
1804 return NULL;
1805 }
1806 EXPORT_IF_KUNIT(amdgpu_dm_get_crtc_by_otg_inst);
1807
1808 /**
1809 * dm_pflip_high_irq() - Handle pageflip interrupt
1810 * @interrupt_params: ignored
1811 *
1812 * Handles the pageflip interrupt by notifying all interested parties
1813 * that the pageflip has been completed.
1814 */
dm_pflip_high_irq(void * interrupt_params)1815 STATIC_IFN_KUNIT void dm_pflip_high_irq(void *interrupt_params)
1816 {
1817 struct amdgpu_crtc *amdgpu_crtc;
1818 struct common_irq_params *irq_params = interrupt_params;
1819 struct amdgpu_device *adev = irq_params->adev;
1820 struct drm_device *dev = adev_to_drm(adev);
1821 unsigned long flags;
1822 struct drm_pending_vblank_event *e;
1823 u32 vpos, hpos, v_blank_start, v_blank_end;
1824 bool vrr_active;
1825
1826 amdgpu_crtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP);
1827
1828 /* IRQ could occur when in initial stage */
1829 /* TODO work and BO cleanup */
1830 if (amdgpu_crtc == NULL) {
1831 drm_dbg_state(dev, "CRTC is null, returning.\n");
1832 return;
1833 }
1834
1835 spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
1836
1837 if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) {
1838 drm_dbg_state(dev,
1839 "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n",
1840 amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED,
1841 amdgpu_crtc->crtc_id, amdgpu_crtc);
1842 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
1843 return;
1844 }
1845
1846 /* page flip completed. */
1847 e = amdgpu_crtc->event;
1848 amdgpu_crtc->event = NULL;
1849
1850 WARN_ON(!e);
1851
1852 vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc);
1853
1854 /* Fixed refresh rate, or VRR scanout position outside front-porch? */
1855 if (!vrr_active ||
1856 !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start,
1857 &v_blank_end, &hpos, &vpos) ||
1858 (vpos < v_blank_start)) {
1859 /* Update to correct count and vblank timestamp if racing with
1860 * vblank irq. This also updates to the correct vblank timestamp
1861 * even in VRR mode, as scanout is past the front-porch atm.
1862 */
1863 drm_crtc_accurate_vblank_count(&amdgpu_crtc->base);
1864
1865 /* Wake up userspace by sending the pageflip event with proper
1866 * count and timestamp of vblank of flip completion.
1867 */
1868 if (e) {
1869 drm_crtc_send_vblank_event(&amdgpu_crtc->base, e);
1870
1871 /* Event sent, so done with vblank for this flip */
1872 drm_crtc_vblank_put(&amdgpu_crtc->base);
1873 }
1874 } else if (e) {
1875 /* VRR active and inside front-porch: vblank count and
1876 * timestamp for pageflip event will only be up to date after
1877 * drm_crtc_handle_vblank() has been executed from late vblank
1878 * irq handler after start of back-porch (vline 0). We queue the
1879 * pageflip event for send-out by drm_crtc_handle_vblank() with
1880 * updated timestamp and count, once it runs after us.
1881 *
1882 * We need to open-code this instead of using the helper
1883 * drm_crtc_arm_vblank_event(), as that helper would
1884 * call drm_crtc_accurate_vblank_count(), which we must
1885 * not call in VRR mode while we are in front-porch!
1886 */
1887
1888 /* sequence will be replaced by real count during send-out. */
1889 e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base);
1890 e->pipe = amdgpu_crtc->crtc_id;
1891
1892 list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list);
1893 e = NULL;
1894 }
1895
1896 /* Keep track of vblank of this flip for flip throttling. We use the
1897 * cooked hw counter, as that one incremented at start of this vblank
1898 * of pageflip completion, so last_flip_vblank is the forbidden count
1899 * for queueing new pageflips if vsync + VRR is enabled.
1900 */
1901 amdgpu_crtc->dm_irq_params.last_flip_vblank =
1902 amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base);
1903
1904 amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
1905 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
1906
1907 drm_dbg_state(dev,
1908 "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n",
1909 amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e);
1910 }
1911 EXPORT_IF_KUNIT(dm_pflip_high_irq);
1912
dm_handle_vmin_vmax_update(struct work_struct * offload_work)1913 STATIC_IFN_KUNIT void dm_handle_vmin_vmax_update(struct work_struct *offload_work)
1914 {
1915 struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work);
1916 struct amdgpu_device *adev = work->adev;
1917 struct dc_stream_state *stream = work->stream;
1918 struct dc_crtc_timing_adjust *adjust = work->adjust;
1919
1920 mutex_lock(&adev->dm.dc_lock);
1921 dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust);
1922 mutex_unlock(&adev->dm.dc_lock);
1923
1924 dc_stream_release(stream);
1925 kfree(work->adjust);
1926 kfree(work);
1927 }
1928 EXPORT_IF_KUNIT(dm_handle_vmin_vmax_update);
1929
schedule_dc_vmin_vmax(struct amdgpu_device * adev,struct dc_stream_state * stream,struct dc_crtc_timing_adjust * adjust)1930 static void schedule_dc_vmin_vmax(struct amdgpu_device *adev,
1931 struct dc_stream_state *stream,
1932 struct dc_crtc_timing_adjust *adjust)
1933 {
1934 struct vupdate_offload_work *offload_work = kzalloc_obj(*offload_work,
1935 GFP_NOWAIT);
1936 if (!offload_work) {
1937 drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n");
1938 return;
1939 }
1940
1941 struct dc_crtc_timing_adjust *adjust_copy = kzalloc_obj(*adjust_copy,
1942 GFP_NOWAIT);
1943 if (!adjust_copy) {
1944 drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n");
1945 kfree(offload_work);
1946 return;
1947 }
1948
1949 dc_stream_retain(stream);
1950 memcpy(adjust_copy, adjust, sizeof(*adjust_copy));
1951
1952 INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update);
1953 offload_work->adev = adev;
1954 offload_work->stream = stream;
1955 offload_work->adjust = adjust_copy;
1956
1957 queue_work(adev->dm.vmin_vmax_wq, &offload_work->work);
1958 }
1959
1960 /**
1961 * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling
1962 * @adev: amdgpu device
1963 * @acrtc: the CRTC to service
1964 * Performs writeback completion, vblank event handling, CRC processing, VRR BTR
1965 * updates and pageflip completion delivery.
1966 *
1967 * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from
1968 * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank
1969 * from dm_crtc_high_irq().
1970 */
dm_crtc_high_irq_handler(struct amdgpu_device * adev,struct amdgpu_crtc * acrtc)1971 static void dm_crtc_high_irq_handler(struct amdgpu_device *adev,
1972 struct amdgpu_crtc *acrtc)
1973 {
1974 unsigned long flags;
1975 int vrr_active;
1976 bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0;
1977
1978 if (acrtc->wb_conn && acrtc->wb_pending) {
1979 if (acrtc->wb_frame_done) {
1980 /*
1981 * Second vblank: the DMA for the captured frame has
1982 * had a full frame period to flush to memory. Signal
1983 * the out fence now.
1984 */
1985 amdgpu_dm_crtc_complete_writeback(acrtc);
1986 } else {
1987 /*
1988 * First vblank after arming: the frame has been
1989 * scanned out and the DMA is finishing. Disable
1990 * writeback immediately to prevent the hardware from
1991 * starting a new capture that would overwrite the
1992 * buffer. Signal completion on the next vblank to
1993 * ensure the DMA is fully flushed to memory.
1994 */
1995 dc_stream_fc_disable_writeback(adev->dm.dc,
1996 acrtc->dm_irq_params.stream, 0);
1997 acrtc->wb_frame_done = true;
1998 }
1999 }
2000
2001 vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
2002
2003 drm_dbg_vbl(adev_to_drm(adev),
2004 "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id,
2005 vrr_active, acrtc->dm_irq_params.active_planes);
2006
2007 /**
2008 * Core vblank handling.
2009 *
2010 * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch
2011 * point, which is the correct place to timestamp both VRR and non-VRR
2012 * vblanks.
2013 *
2014 * On DCE this handler runs at the start of front-porch, where only
2015 * non-VRR timestamping is valid; VRR vblank is deferred to
2016 * dm_vupdate_high_irq() after end of front-porch.
2017 */
2018 if (is_dcn || !vrr_active)
2019 amdgpu_dm_crtc_handle_vblank(acrtc);
2020
2021 /**
2022 * Following stuff must happen at start of vblank, for crc
2023 * computation and below-the-range btr support in vrr mode.
2024 */
2025 amdgpu_dm_crtc_handle_crc_irq(&acrtc->base);
2026
2027 /* BTR updates need to happen before VUPDATE on Vega and above. */
2028 if (adev->family < AMDGPU_FAMILY_AI)
2029 return;
2030
2031 spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
2032
2033 if (acrtc->dm_irq_params.stream &&
2034 acrtc->dm_irq_params.vrr_params.supported) {
2035 bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
2036 bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
2037 bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE;
2038
2039 mod_freesync_handle_v_update(adev->dm.freesync_module,
2040 acrtc->dm_irq_params.stream,
2041 &acrtc->dm_irq_params.vrr_params);
2042
2043 /* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */
2044 if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
2045 schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream,
2046 &acrtc->dm_irq_params.vrr_params.adjust);
2047 }
2048 }
2049
2050 /*
2051 * Deliver pageflip completion events (DCN only).
2052 *
2053 * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch
2054 * point. Deliver any pending pageflip completion event from here,
2055 * once HW has consumed the new address (the OTG no longer reports a
2056 * pending flip).
2057 *
2058 * Also handle the case here where there aren't any active planes and
2059 * DCN HUBP may be clock-gated, so the flip-pending status may be
2060 * undefined.
2061 */
2062 if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
2063 acrtc->event) {
2064
2065 if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) {
2066 drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
2067 acrtc->event = NULL;
2068 drm_crtc_vblank_put(&acrtc->base);
2069 acrtc->pflip_status = AMDGPU_FLIP_NONE;
2070 }
2071 /*
2072 * If the flip is still pending, leave it armed and
2073 * retry on the next vupdate.
2074 */
2075 } else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
2076 acrtc->dm_irq_params.active_planes == 0) {
2077
2078 if (acrtc->event) {
2079 drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
2080 acrtc->event = NULL;
2081 drm_crtc_vblank_put(&acrtc->base);
2082 }
2083 acrtc->pflip_status = AMDGPU_FLIP_NONE;
2084 }
2085
2086 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
2087 }
2088
dm_vupdate_high_irq(void * interrupt_params)2089 STATIC_IFN_KUNIT void dm_vupdate_high_irq(void *interrupt_params)
2090 {
2091 struct common_irq_params *irq_params = interrupt_params;
2092 struct amdgpu_device *adev = irq_params->adev;
2093 struct amdgpu_crtc *acrtc;
2094 struct drm_device *drm_dev;
2095 struct drm_vblank_crtc *vblank;
2096 ktime_t frame_duration_ns, previous_timestamp;
2097 unsigned long flags;
2098 int vrr_active;
2099
2100 acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
2101 if (!acrtc)
2102 return;
2103
2104 vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
2105 drm_dev = acrtc->base.dev;
2106 vblank = drm_crtc_vblank_crtc(&acrtc->base);
2107 previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
2108 frame_duration_ns = vblank->time - previous_timestamp;
2109
2110 if (frame_duration_ns > 0) {
2111 trace_amdgpu_refresh_rate_track(acrtc->base.index,
2112 frame_duration_ns,
2113 ktime_divns(NSEC_PER_SEC, frame_duration_ns));
2114 atomic64_set(&irq_params->previous_timestamp, vblank->time);
2115 }
2116
2117 drm_dbg_vbl(drm_dev,
2118 "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
2119 vrr_active);
2120
2121 /*
2122 * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver
2123 * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are
2124 * not used. Run the full handler here.
2125 */
2126 if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) {
2127 dm_crtc_high_irq_handler(adev, acrtc);
2128 return;
2129 }
2130
2131 /* DCE only below. */
2132
2133 /* Core vblank handling is done here after end of front-porch in
2134 * vrr mode, as vblank timestamping will give valid results
2135 * while now done after front-porch. This will also deliver
2136 * page-flip completion events that have been queued to us
2137 * if a pageflip happened inside front-porch.
2138 */
2139 if (vrr_active && acrtc->dm_irq_params.stream) {
2140 bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
2141 bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
2142 bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state
2143 == VRR_STATE_ACTIVE_VARIABLE;
2144
2145 amdgpu_dm_crtc_handle_vblank(acrtc);
2146
2147 /* BTR processing for pre-DCE12 ASICs */
2148 if (adev->family < AMDGPU_FAMILY_AI) {
2149 spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
2150 mod_freesync_handle_v_update(
2151 adev->dm.freesync_module,
2152 acrtc->dm_irq_params.stream,
2153 &acrtc->dm_irq_params.vrr_params);
2154
2155 if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
2156 schedule_dc_vmin_vmax(adev,
2157 acrtc->dm_irq_params.stream,
2158 &acrtc->dm_irq_params.vrr_params.adjust);
2159 }
2160 spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
2161 }
2162 }
2163 }
2164 EXPORT_IF_KUNIT(dm_vupdate_high_irq);
2165
2166 /**
2167 * dm_crtc_high_irq() - Handles CRTC interrupt
2168 * @interrupt_params: used for determining the CRTC instance
2169 *
2170 * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler.
2171 *
2172 * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is
2173 * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq().
2174 */
dm_crtc_high_irq(void * interrupt_params)2175 STATIC_IFN_KUNIT void dm_crtc_high_irq(void *interrupt_params)
2176 {
2177 struct common_irq_params *irq_params = interrupt_params;
2178 struct amdgpu_device *adev = irq_params->adev;
2179 struct amdgpu_crtc *acrtc;
2180
2181 acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
2182 if (!acrtc)
2183 return;
2184
2185 dm_crtc_high_irq_handler(adev, acrtc);
2186 }
2187 EXPORT_IF_KUNIT(dm_crtc_high_irq);
2188
2189 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
2190 /**
2191 * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for
2192 * DCN generation ASICs
2193 * @interrupt_params: interrupt parameters
2194 *
2195 * Used to set crc window/read out crc value at vertical line 0 position
2196 */
dm_dcn_vertical_interrupt0_high_irq(void * interrupt_params)2197 static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params)
2198 {
2199 struct common_irq_params *irq_params = interrupt_params;
2200 struct amdgpu_device *adev = irq_params->adev;
2201 struct amdgpu_crtc *acrtc;
2202
2203 acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0);
2204
2205 if (!acrtc)
2206 return;
2207
2208 amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);
2209 }
2210 #endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */
2211
dm_handle_hpd_work(struct work_struct * work)2212 STATIC_IFN_KUNIT void dm_handle_hpd_work(struct work_struct *work)
2213 {
2214 struct dmub_hpd_work *dmub_hpd_wrk;
2215
2216 dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work);
2217
2218 if (!dmub_hpd_wrk->dmub_notify) {
2219 drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL");
2220 return;
2221 }
2222
2223 if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) {
2224 dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev,
2225 dmub_hpd_wrk->dmub_notify);
2226 }
2227
2228 kfree(dmub_hpd_wrk->dmub_notify);
2229 kfree(dmub_hpd_wrk);
2230
2231 }
2232 EXPORT_IF_KUNIT(dm_handle_hpd_work);
2233
dmub_notification_type_str(enum dmub_notification_type e)2234 STATIC_IFN_KUNIT const char *dmub_notification_type_str(enum dmub_notification_type e)
2235 {
2236 switch (e) {
2237 case DMUB_NOTIFICATION_NO_DATA:
2238 return "NO_DATA";
2239 case DMUB_NOTIFICATION_AUX_REPLY:
2240 return "AUX_REPLY";
2241 case DMUB_NOTIFICATION_HPD:
2242 return "HPD";
2243 case DMUB_NOTIFICATION_HPD_IRQ:
2244 return "HPD_IRQ";
2245 case DMUB_NOTIFICATION_SET_CONFIG_REPLY:
2246 return "SET_CONFIG_REPLY";
2247 case DMUB_NOTIFICATION_DPIA_NOTIFICATION:
2248 return "DPIA_NOTIFICATION";
2249 case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY:
2250 return "HPD_SENSE_NOTIFY";
2251 case DMUB_NOTIFICATION_FUSED_IO:
2252 return "FUSED_IO";
2253 default:
2254 return "<unknown>";
2255 }
2256 }
2257 EXPORT_IF_KUNIT(dmub_notification_type_str);
2258
2259 #define DMUB_TRACE_MAX_READ 64
2260 /**
2261 * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt
2262 * @interrupt_params: used for determining the Outbox instance
2263 *
2264 * Handles the Outbox Interrupt
2265 * event handler.
2266 */
dm_dmub_outbox1_low_irq(void * interrupt_params)2267 STATIC_IFN_KUNIT void dm_dmub_outbox1_low_irq(void *interrupt_params)
2268 {
2269 struct dmub_notification notify = {0};
2270 struct common_irq_params *irq_params = interrupt_params;
2271 struct amdgpu_device *adev = irq_params->adev;
2272 struct amdgpu_display_manager *dm = &adev->dm;
2273 struct dmcub_trace_buf_entry entry = { 0 };
2274 u32 count = 0;
2275 struct dmub_hpd_work *dmub_hpd_wrk;
2276
2277 do {
2278 if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
2279 trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
2280 entry.param0, entry.param1);
2281
2282 drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
2283 entry.trace_code, entry.tick_count, entry.param0, entry.param1);
2284 } else
2285 break;
2286
2287 count++;
2288
2289 } while (count <= DMUB_TRACE_MAX_READ);
2290
2291 if (count > DMUB_TRACE_MAX_READ)
2292 drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ");
2293
2294 if (dc_enable_dmub_notifications(adev->dm.dc) &&
2295 irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) {
2296
2297 do {
2298 dc_stat_get_dmub_notification(adev->dm.dc, ¬ify);
2299 if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) {
2300 drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type);
2301 continue;
2302 }
2303 if (!dm->dmub_callback[notify.type]) {
2304 drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n",
2305 dmub_notification_type_str(notify.type));
2306 continue;
2307 }
2308 if (dm->dmub_thread_offload[notify.type]) {
2309 dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk,
2310 GFP_ATOMIC);
2311 if (!dmub_hpd_wrk) {
2312 drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk");
2313 return;
2314 }
2315 dmub_hpd_wrk->dmub_notify = kmemdup(¬ify, sizeof(struct dmub_notification),
2316 GFP_ATOMIC);
2317 if (!dmub_hpd_wrk->dmub_notify) {
2318 kfree(dmub_hpd_wrk);
2319 drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify");
2320 return;
2321 }
2322 INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work);
2323 dmub_hpd_wrk->adev = adev;
2324 queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work);
2325 } else {
2326 dm->dmub_callback[notify.type](adev, ¬ify);
2327 }
2328 } while (notify.pending_notification);
2329 }
2330 }
2331 EXPORT_IF_KUNIT(dm_dmub_outbox1_low_irq);
2332
2333 /* Register IRQ sources and initialize IRQ callbacks */
amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device * adev)2334 int amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device *adev)
2335 {
2336 struct dc *dc = adev->dm.dc;
2337 struct common_irq_params *c_irq_params;
2338 struct dc_interrupt_params int_params = {0};
2339 int r;
2340 int i;
2341 unsigned int src_id;
2342 unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
2343 /* Use different interrupts for VBLANK on DCE 6 vs. newer. */
2344 const unsigned int vblank_d1 =
2345 adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0
2346 ? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1;
2347
2348 if (adev->family >= AMDGPU_FAMILY_AI)
2349 client_id = SOC15_IH_CLIENTID_DCE;
2350
2351 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
2352 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
2353
2354 /*
2355 * Actions of amdgpu_irq_add_id():
2356 * 1. Register a set() function with base driver.
2357 * Base driver will call set() function to enable/disable an
2358 * interrupt in DC hardware.
2359 * 2. Register amdgpu_dm_irq_handler().
2360 * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
2361 * coming from DC hardware.
2362 * amdgpu_dm_irq_handler() will re-direct the interrupt to DC
2363 * for acknowledging and handling.
2364 */
2365
2366 /* Use VBLANK interrupt */
2367 for (i = 0; i < adev->mode_info.num_crtc; i++) {
2368 src_id = vblank_d1 + i;
2369 r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq);
2370 if (r) {
2371 drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
2372 return r;
2373 }
2374
2375 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2376 int_params.irq_source =
2377 dc_interrupt_to_irq_source(dc, src_id, 0);
2378
2379 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
2380 int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 ||
2381 int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) {
2382 drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
2383 return -EINVAL;
2384 }
2385
2386 c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
2387
2388 c_irq_params->adev = adev;
2389 c_irq_params->irq_src = int_params.irq_source;
2390
2391 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2392 dm_crtc_high_irq, c_irq_params))
2393 return -ENOMEM;
2394 }
2395
2396 if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
2397 /* Use VUPDATE interrupt */
2398 for (i = 0; i < adev->mode_info.num_crtc; i++) {
2399 src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2;
2400 r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq);
2401 if (r) {
2402 drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
2403 return r;
2404 }
2405
2406 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2407 int_params.irq_source =
2408 dc_interrupt_to_irq_source(dc, src_id, 0);
2409
2410 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
2411 int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 ||
2412 int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) {
2413 drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
2414 return -EINVAL;
2415 }
2416
2417 c_irq_params = &adev->dm.vupdate_params[
2418 int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
2419 c_irq_params->adev = adev;
2420 c_irq_params->irq_src = int_params.irq_source;
2421
2422 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2423 dm_vupdate_high_irq, c_irq_params))
2424 return -ENOMEM;
2425 }
2426 }
2427
2428 /* Use GRPH_PFLIP interrupt */
2429 for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP;
2430 i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) {
2431 r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq);
2432 if (r) {
2433 drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
2434 return r;
2435 }
2436
2437 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2438 int_params.irq_source =
2439 dc_interrupt_to_irq_source(dc, i, 0);
2440
2441 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
2442 int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST ||
2443 int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) {
2444 drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
2445 return -EINVAL;
2446 }
2447
2448 c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
2449
2450 c_irq_params->adev = adev;
2451 c_irq_params->irq_src = int_params.irq_source;
2452
2453 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2454 dm_pflip_high_irq, c_irq_params))
2455 return -ENOMEM;
2456 }
2457
2458 /* HPD */
2459 r = amdgpu_irq_add_id(adev, client_id,
2460 VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
2461 if (r) {
2462 drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
2463 return r;
2464 }
2465
2466 r = amdgpu_dm_register_hpd_handlers(adev);
2467
2468 return r;
2469 }
2470 EXPORT_IF_KUNIT(amdgpu_dm_dce110_register_irq_handlers);
2471
2472 /* Register IRQ sources and initialize IRQ callbacks */
amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device * adev)2473 int amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device *adev)
2474 {
2475 struct dc *dc = adev->dm.dc;
2476 struct common_irq_params *c_irq_params;
2477 struct dc_interrupt_params int_params = {0};
2478 int r;
2479 int i;
2480 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
2481 static const unsigned int vrtl_int_srcid[] = {
2482 DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL,
2483 DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL,
2484 DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL,
2485 DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL,
2486 DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL,
2487 DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL
2488 };
2489 #endif
2490
2491 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
2492 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
2493
2494 /*
2495 * Actions of amdgpu_irq_add_id():
2496 * 1. Register a set() function with base driver.
2497 * Base driver will call set() function to enable/disable an
2498 * interrupt in DC hardware.
2499 * 2. Register amdgpu_dm_irq_handler().
2500 * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
2501 * coming from DC hardware.
2502 * amdgpu_dm_irq_handler() will re-direct the interrupt to DC
2503 * for acknowledging and handling.
2504 */
2505
2506 /* Use otg vertical line interrupt */
2507 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
2508 for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) {
2509 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE,
2510 vrtl_int_srcid[i], &adev->vline0_irq);
2511
2512 if (r) {
2513 drm_err(adev_to_drm(adev), "Failed to add vline0 irq id!\n");
2514 return r;
2515 }
2516
2517 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2518 int_params.irq_source =
2519 dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0);
2520
2521 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
2522 int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 ||
2523 int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) {
2524 drm_err(adev_to_drm(adev), "Failed to register vline0 irq!\n");
2525 return -EINVAL;
2526 }
2527
2528 c_irq_params = &adev->dm.vline0_params[int_params.irq_source
2529 - DC_IRQ_SOURCE_DC1_VLINE0];
2530
2531 c_irq_params->adev = adev;
2532 c_irq_params->irq_src = int_params.irq_source;
2533
2534 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2535 dm_dcn_vertical_interrupt0_high_irq,
2536 c_irq_params))
2537 return -ENOMEM;
2538 }
2539 #endif
2540
2541 /* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to
2542 * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx
2543 * to trigger at end of each vblank, regardless of state of the lock,
2544 * matching DCE behaviour.
2545 */
2546 for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT;
2547 i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1;
2548 i++) {
2549 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq);
2550
2551 if (r) {
2552 drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
2553 return r;
2554 }
2555
2556 int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2557 int_params.irq_source =
2558 dc_interrupt_to_irq_source(dc, i, 0);
2559
2560 if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
2561 int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 ||
2562 int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) {
2563 drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
2564 return -EINVAL;
2565 }
2566
2567 c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
2568
2569 c_irq_params->adev = adev;
2570 c_irq_params->irq_src = int_params.irq_source;
2571
2572 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2573 dm_vupdate_high_irq, c_irq_params))
2574 return -ENOMEM;
2575 }
2576
2577 /* HPD */
2578 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
2579 &adev->hpd_irq);
2580 if (r) {
2581 drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
2582 return r;
2583 }
2584
2585 r = amdgpu_dm_register_hpd_handlers(adev);
2586
2587 return r;
2588 }
2589 EXPORT_IF_KUNIT(amdgpu_dm_dcn10_register_irq_handlers);
2590
2591 /* Register Outbox IRQ sources and initialize IRQ callbacks */
amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device * adev)2592 int amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device *adev)
2593 {
2594 struct dc *dc = adev->dm.dc;
2595 struct common_irq_params *c_irq_params;
2596 struct dc_interrupt_params int_params = {0};
2597 int r, i;
2598
2599 int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
2600 int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
2601
2602 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT,
2603 &adev->dmub_outbox_irq);
2604 if (r) {
2605 drm_err(adev_to_drm(adev), "Failed to add outbox irq id!\n");
2606 return r;
2607 }
2608
2609 if (dc->ctx->dmub_srv) {
2610 i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT;
2611 int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
2612 int_params.irq_source =
2613 dc_interrupt_to_irq_source(dc, i, 0);
2614
2615 c_irq_params = &adev->dm.dmub_outbox_params[0];
2616
2617 c_irq_params->adev = adev;
2618 c_irq_params->irq_src = int_params.irq_source;
2619
2620 if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2621 dm_dmub_outbox1_low_irq, c_irq_params))
2622 return -ENOMEM;
2623 }
2624
2625 return 0;
2626 }
2627 EXPORT_IF_KUNIT(amdgpu_dm_register_outbox_irq_handlers);
2628