1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright (C) 2024 Intel Corporation 4 */ 5 6 #include <linux/kernel.h> 7 8 #include <drm/drm_print.h> 9 10 #include "intel_de.h" 11 #include "intel_display_regs.h" 12 #include "intel_dmc_regs.h" 13 #include "intel_dmc_wl.h" 14 15 /** 16 * DOC: DMC wakelock support 17 * 18 * Wake lock is the mechanism to cause display engine to exit DC 19 * states to allow programming to registers that are powered down in 20 * those states. Previous projects exited DC states automatically when 21 * detecting programming. Now software controls the exit by 22 * programming the wake lock. This improves system performance and 23 * system interactions and better fits the flip queue style of 24 * programming. Wake lock is only required when DC5, DC6, or DC6v have 25 * been enabled in DC_STATE_EN and the wake lock mode of operation has 26 * been enabled. 27 * 28 * The wakelock mechanism in DMC allows the display engine to exit DC 29 * states explicitly before programming registers that may be powered 30 * down. In earlier hardware, this was done automatically and 31 * implicitly when the display engine accessed a register. With the 32 * wakelock implementation, the driver asserts a wakelock in DMC, 33 * which forces it to exit the DC state until the wakelock is 34 * deasserted. 35 * 36 * The mechanism can be enabled and disabled by writing to the 37 * DMC_WAKELOCK_CFG register. There are also 13 control registers 38 * that can be used to hold and release different wakelocks. In the 39 * current implementation, we only need one wakelock, so only 40 * DMC_WAKELOCK1_CTL is used. The other definitions are here for 41 * potential future use. 42 * 43 * This is available starting with Xe2_LPD (display version 20) as an 44 * experimental feature and on Xe3_LPD (display version 30) as the 45 * first display release with official support. That means that we 46 * only enable the feature by default on the latter and using it on 47 * the former requires explicitly using the enable_dmc_wl module 48 * parameter. 49 */ 50 51 /* 52 * Define DMC_WAKELOCK_CTL_TIMEOUT_US in microseconds because we use the 53 * atomic variant of waiting MMIO. 54 */ 55 #define DMC_WAKELOCK_CTL_TIMEOUT_US 5000 56 #define DMC_WAKELOCK_HOLD_TIME 5 57 58 /* 59 * Possible non-negative values for the enable_dmc_wl param. 60 */ 61 enum { 62 ENABLE_DMC_WL_DISABLED, 63 ENABLE_DMC_WL_ENABLED, 64 ENABLE_DMC_WL_ANY_REGISTER, 65 ENABLE_DMC_WL_ALWAYS_LOCKED, 66 ENABLE_DMC_WL_MAX, 67 }; 68 69 struct intel_dmc_wl_range { 70 u32 start; 71 u32 end; 72 }; 73 74 static const struct intel_dmc_wl_range powered_off_ranges[] = { 75 { .start = 0x44400, .end = 0x4447f }, /* PIPE interrupt registers */ 76 { .start = 0x60000, .end = 0x7ffff }, 77 {}, 78 }; 79 80 static const struct intel_dmc_wl_range xe3lpd_dc5_dc6_dmc_ranges[] = { 81 { .start = 0x45500 }, /* DC_STATE_SEL */ 82 { .start = 0x457a0, .end = 0x457b0 }, /* DC*_RESIDENCY_COUNTER */ 83 { .start = 0x45504 }, /* DC_STATE_EN */ 84 { .start = 0x45400, .end = 0x4540c }, /* PWR_WELL_CTL_* */ 85 { .start = 0x454f0 }, /* RETENTION_CTRL */ 86 87 /* DBUF_CTL_* */ 88 { .start = 0x44300 }, 89 { .start = 0x44304 }, 90 { .start = 0x44f00 }, 91 { .start = 0x44f04 }, 92 { .start = 0x44fe8 }, 93 { .start = 0x45008 }, 94 95 { .start = 0x46070 }, /* CDCLK_PLL_ENABLE */ 96 { .start = 0x46000 }, /* CDCLK_CTL */ 97 { .start = 0x46008 }, /* CDCLK_SQUASH_CTL */ 98 99 /* TRANS_CMTG_CTL_* */ 100 { .start = 0x6fa88 }, 101 { .start = 0x6fb88 }, 102 103 { .start = 0x46430 }, /* CHICKEN_DCPR_1 */ 104 { .start = 0x46434 }, /* CHICKEN_DCPR_2 */ 105 { .start = 0x454a0 }, /* CHICKEN_DCPR_4 */ 106 { .start = 0x42084 }, /* CHICKEN_MISC_2 */ 107 { .start = 0x42088 }, /* CHICKEN_MISC_3 */ 108 { .start = 0x46160 }, /* CMTG_CLK_SEL */ 109 { .start = 0x8f000, .end = 0x8ffff }, /* Main DMC registers */ 110 { .start = 0x45230 }, /* INITIATE_PM_DMD_REQ */ 111 112 {}, 113 }; 114 115 static const struct intel_dmc_wl_range xe3lpd_dc3co_dmc_ranges[] = { 116 { .start = 0x454a0 }, /* CHICKEN_DCPR_4 */ 117 118 { .start = 0x45504 }, /* DC_STATE_EN */ 119 120 /* DBUF_CTL_* */ 121 { .start = 0x44300 }, 122 { .start = 0x44304 }, 123 { .start = 0x44f00 }, 124 { .start = 0x44f04 }, 125 { .start = 0x44fe8 }, 126 { .start = 0x45008 }, 127 128 { .start = 0x46070 }, /* CDCLK_PLL_ENABLE */ 129 { .start = 0x46000 }, /* CDCLK_CTL */ 130 { .start = 0x46008 }, /* CDCLK_SQUASH_CTL */ 131 { .start = 0x8f000, .end = 0x8ffff }, /* Main DMC registers */ 132 133 /* Scanline registers */ 134 { .start = 0x70000 }, 135 { .start = 0x70004 }, 136 { .start = 0x70014 }, 137 { .start = 0x70018 }, 138 { .start = 0x71000 }, 139 { .start = 0x71004 }, 140 { .start = 0x71014 }, 141 { .start = 0x71018 }, 142 { .start = 0x72000 }, 143 { .start = 0x72004 }, 144 { .start = 0x72014 }, 145 { .start = 0x72018 }, 146 { .start = 0x73000 }, 147 { .start = 0x73004 }, 148 { .start = 0x73014 }, 149 { .start = 0x73018 }, 150 { .start = 0x7b000 }, 151 { .start = 0x7b004 }, 152 { .start = 0x7b014 }, 153 { .start = 0x7b018 }, 154 { .start = 0x7c000 }, 155 { .start = 0x7c004 }, 156 { .start = 0x7c014 }, 157 { .start = 0x7c018 }, 158 159 {}, 160 }; 161 162 static void __intel_dmc_wl_release(struct intel_display *display) 163 { 164 struct intel_dmc_wl *wl = &display->wl; 165 166 WARN_ON(refcount_read(&wl->refcount)); 167 168 queue_delayed_work(display->wq.unordered, &wl->work, 169 msecs_to_jiffies(DMC_WAKELOCK_HOLD_TIME)); 170 } 171 172 static void intel_dmc_wl_work(struct work_struct *work) 173 { 174 struct intel_dmc_wl *wl = 175 container_of(work, struct intel_dmc_wl, work.work); 176 struct intel_display *display = 177 container_of(wl, struct intel_display, wl); 178 unsigned long flags; 179 180 spin_lock_irqsave(&wl->lock, flags); 181 182 /* 183 * Bail out if refcount became non-zero while waiting for the spinlock, 184 * meaning that the lock is now taken again. 185 */ 186 if (refcount_read(&wl->refcount)) 187 goto out_unlock; 188 189 intel_de_rmw_fw(display, DMC_WAKELOCK1_CTL, DMC_WAKELOCK_CTL_REQ, 0); 190 191 if (intel_de_wait_fw_us_atomic(display, DMC_WAKELOCK1_CTL, 192 DMC_WAKELOCK_CTL_ACK, 0, 193 DMC_WAKELOCK_CTL_TIMEOUT_US, NULL)) { 194 WARN_RATELIMIT(1, "DMC wakelock release timed out"); 195 goto out_unlock; 196 } 197 198 wl->taken = false; 199 200 out_unlock: 201 spin_unlock_irqrestore(&wl->lock, flags); 202 } 203 204 static void __intel_dmc_wl_take(struct intel_display *display) 205 { 206 struct intel_dmc_wl *wl = &display->wl; 207 208 /* 209 * Only try to take the wakelock if it's not marked as taken 210 * yet. It may be already taken at this point if we have 211 * already released the last reference, but the work has not 212 * run yet. 213 */ 214 if (wl->taken) 215 return; 216 217 intel_de_rmw_fw(display, DMC_WAKELOCK1_CTL, 0, DMC_WAKELOCK_CTL_REQ); 218 219 /* 220 * We need to use the atomic variant of the waiting routine 221 * because the DMC wakelock is also taken in atomic context. 222 */ 223 if (intel_de_wait_fw_us_atomic(display, DMC_WAKELOCK1_CTL, 224 DMC_WAKELOCK_CTL_ACK, 225 DMC_WAKELOCK_CTL_ACK, 226 DMC_WAKELOCK_CTL_TIMEOUT_US, NULL)) { 227 WARN_RATELIMIT(1, "DMC wakelock ack timed out"); 228 return; 229 } 230 231 wl->taken = true; 232 } 233 234 static bool intel_dmc_wl_reg_in_range(intel_reg_t reg, 235 const struct intel_dmc_wl_range ranges[]) 236 { 237 u32 offset = intel_reg_offset(reg); 238 239 for (int i = 0; ranges[i].start; i++) { 240 u32 end = ranges[i].end ?: ranges[i].start; 241 242 if (ranges[i].start <= offset && offset <= end) 243 return true; 244 } 245 246 return false; 247 } 248 249 static bool intel_dmc_wl_check_range(struct intel_display *display, 250 intel_reg_t reg, 251 u32 dc_state) 252 { 253 const struct intel_dmc_wl_range *ranges; 254 255 if (display->params.enable_dmc_wl == ENABLE_DMC_WL_ANY_REGISTER) 256 return true; 257 258 /* 259 * Check that the offset is in one of the ranges for which 260 * registers are powered off during DC states. 261 */ 262 if (intel_dmc_wl_reg_in_range(reg, powered_off_ranges)) 263 return true; 264 265 /* 266 * Check that the offset is for a register that is touched by 267 * the DMC and requires a DC exit for proper access. 268 */ 269 switch (dc_state) { 270 case DC_STATE_EN_DC3CO: 271 ranges = xe3lpd_dc3co_dmc_ranges; 272 break; 273 case DC_STATE_EN_UPTO_DC5: 274 case DC_STATE_EN_UPTO_DC6: 275 ranges = xe3lpd_dc5_dc6_dmc_ranges; 276 break; 277 default: 278 ranges = NULL; 279 } 280 281 if (ranges && intel_dmc_wl_reg_in_range(reg, ranges)) 282 return true; 283 284 return false; 285 } 286 287 static bool __intel_dmc_wl_supported(struct intel_display *display) 288 { 289 return display->params.enable_dmc_wl; 290 } 291 292 static void intel_dmc_wl_sanitize_param(struct intel_display *display) 293 { 294 const char *desc; 295 296 if (DISPLAY_VER(display) < 20) { 297 display->params.enable_dmc_wl = ENABLE_DMC_WL_DISABLED; 298 } else if (display->params.enable_dmc_wl < 0) { 299 if (DISPLAY_VER(display) >= 30) 300 display->params.enable_dmc_wl = ENABLE_DMC_WL_ENABLED; 301 else 302 display->params.enable_dmc_wl = ENABLE_DMC_WL_DISABLED; 303 } else if (display->params.enable_dmc_wl >= ENABLE_DMC_WL_MAX) { 304 display->params.enable_dmc_wl = ENABLE_DMC_WL_ENABLED; 305 } 306 307 drm_WARN_ON(display->drm, 308 display->params.enable_dmc_wl < 0 || 309 display->params.enable_dmc_wl >= ENABLE_DMC_WL_MAX); 310 311 switch (display->params.enable_dmc_wl) { 312 case ENABLE_DMC_WL_DISABLED: 313 desc = "disabled"; 314 break; 315 case ENABLE_DMC_WL_ENABLED: 316 desc = "enabled"; 317 break; 318 case ENABLE_DMC_WL_ANY_REGISTER: 319 desc = "match any register"; 320 break; 321 case ENABLE_DMC_WL_ALWAYS_LOCKED: 322 desc = "always locked"; 323 break; 324 default: 325 desc = "unknown"; 326 break; 327 } 328 329 drm_dbg_kms(display->drm, "Sanitized enable_dmc_wl value: %d (%s)\n", 330 display->params.enable_dmc_wl, desc); 331 } 332 333 void intel_dmc_wl_init(struct intel_display *display) 334 { 335 struct intel_dmc_wl *wl = &display->wl; 336 337 intel_dmc_wl_sanitize_param(display); 338 339 if (!display->params.enable_dmc_wl) 340 return; 341 342 INIT_DELAYED_WORK(&wl->work, intel_dmc_wl_work); 343 spin_lock_init(&wl->lock); 344 refcount_set(&wl->refcount, 345 display->params.enable_dmc_wl == ENABLE_DMC_WL_ALWAYS_LOCKED ? 1 : 0); 346 } 347 348 /* Must only be called as part of enabling dynamic DC states. */ 349 void intel_dmc_wl_enable(struct intel_display *display, u32 dc_state) 350 { 351 struct intel_dmc_wl *wl = &display->wl; 352 unsigned long flags; 353 354 if (!__intel_dmc_wl_supported(display)) 355 return; 356 357 spin_lock_irqsave(&wl->lock, flags); 358 359 wl->dc_state = dc_state; 360 361 if (drm_WARN_ON(display->drm, wl->enabled)) 362 goto out_unlock; 363 364 /* 365 * Enable wakelock in DMC. We shouldn't try to take the 366 * wakelock, because we're just enabling it, so call the 367 * non-locking version directly here. 368 */ 369 intel_de_rmw_fw(display, DMC_WAKELOCK_CFG, 0, DMC_WAKELOCK_CFG_ENABLE); 370 371 wl->enabled = true; 372 373 /* 374 * This would be racy in the following scenario: 375 * 376 * 1. Function A calls intel_dmc_wl_get(); 377 * 2. Some function calls intel_dmc_wl_disable(); 378 * 3. Some function calls intel_dmc_wl_enable(); 379 * 4. Concurrently with (3), function A performs the MMIO in between 380 * setting DMC_WAKELOCK_CFG_ENABLE and asserting the lock with 381 * __intel_dmc_wl_take(). 382 * 383 * TODO: Check with the hardware team whether it is safe to assert the 384 * hardware lock before enabling to avoid such a scenario. Otherwise, we 385 * would need to deal with it via software synchronization. 386 */ 387 if (refcount_read(&wl->refcount)) 388 __intel_dmc_wl_take(display); 389 390 out_unlock: 391 spin_unlock_irqrestore(&wl->lock, flags); 392 } 393 394 /* Must only be called as part of disabling dynamic DC states. */ 395 void intel_dmc_wl_disable(struct intel_display *display) 396 { 397 struct intel_dmc_wl *wl = &display->wl; 398 unsigned long flags; 399 400 if (!__intel_dmc_wl_supported(display)) 401 return; 402 403 intel_dmc_wl_flush_release_work(display); 404 405 spin_lock_irqsave(&wl->lock, flags); 406 407 if (drm_WARN_ON(display->drm, !wl->enabled)) 408 goto out_unlock; 409 410 /* Disable wakelock in DMC */ 411 intel_de_rmw_fw(display, DMC_WAKELOCK_CFG, DMC_WAKELOCK_CFG_ENABLE, 0); 412 413 wl->enabled = false; 414 415 /* 416 * The spec is not explicit about the expectation of existing 417 * lock users at the moment of disabling, but it does say that we must 418 * clear DMC_WAKELOCK_CTL_REQ, which gives us a clue that it is okay to 419 * disable with existing lock users. 420 * 421 * TODO: Get the correct expectation from the hardware team. 422 */ 423 intel_de_rmw_fw(display, DMC_WAKELOCK1_CTL, DMC_WAKELOCK_CTL_REQ, 0); 424 425 wl->taken = false; 426 427 out_unlock: 428 spin_unlock_irqrestore(&wl->lock, flags); 429 } 430 431 void intel_dmc_wl_flush_release_work(struct intel_display *display) 432 { 433 struct intel_dmc_wl *wl = &display->wl; 434 435 if (!__intel_dmc_wl_supported(display)) 436 return; 437 438 flush_delayed_work(&wl->work); 439 } 440 441 void intel_dmc_wl_get(struct intel_display *display, intel_reg_t reg) 442 { 443 struct intel_dmc_wl *wl = &display->wl; 444 unsigned long flags; 445 446 if (!__intel_dmc_wl_supported(display)) 447 return; 448 449 spin_lock_irqsave(&wl->lock, flags); 450 451 if (intel_reg_valid(reg) && 452 !intel_dmc_wl_check_range(display, reg, wl->dc_state)) 453 goto out_unlock; 454 455 if (!wl->enabled) { 456 if (!refcount_inc_not_zero(&wl->refcount)) 457 refcount_set(&wl->refcount, 1); 458 goto out_unlock; 459 } 460 461 cancel_delayed_work(&wl->work); 462 463 if (refcount_inc_not_zero(&wl->refcount)) 464 goto out_unlock; 465 466 refcount_set(&wl->refcount, 1); 467 468 __intel_dmc_wl_take(display); 469 470 out_unlock: 471 spin_unlock_irqrestore(&wl->lock, flags); 472 } 473 474 void intel_dmc_wl_put(struct intel_display *display, intel_reg_t reg) 475 { 476 struct intel_dmc_wl *wl = &display->wl; 477 unsigned long flags; 478 479 if (!__intel_dmc_wl_supported(display)) 480 return; 481 482 spin_lock_irqsave(&wl->lock, flags); 483 484 if (intel_reg_valid(reg) && 485 !intel_dmc_wl_check_range(display, reg, wl->dc_state)) 486 goto out_unlock; 487 488 if (WARN_RATELIMIT(!refcount_read(&wl->refcount), 489 "Tried to put wakelock with refcount zero\n")) 490 goto out_unlock; 491 492 if (refcount_dec_and_test(&wl->refcount)) { 493 if (!wl->enabled) 494 goto out_unlock; 495 496 __intel_dmc_wl_release(display); 497 498 goto out_unlock; 499 } 500 501 out_unlock: 502 spin_unlock_irqrestore(&wl->lock, flags); 503 } 504 505 void intel_dmc_wl_get_noreg(struct intel_display *display) 506 { 507 intel_dmc_wl_get(display, INVALID_MMIO_REG); 508 } 509 510 void intel_dmc_wl_put_noreg(struct intel_display *display) 511 { 512 intel_dmc_wl_put(display, INVALID_MMIO_REG); 513 } 514