1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 #include <linux/kernel.h> 8 9 #include <drm/drm_print.h> 10 11 #include "gt/intel_gt_regs.h" 12 13 #include "i915_drv.h" 14 #include "i915_reg.h" 15 #include "i915_trace.h" 16 #include "i915_utils.h" 17 #include "i915_wait_util.h" 18 #include "intel_clock_gating.h" 19 #include "intel_uncore_trace.h" 20 #include "vlv_suspend.h" 21 22 struct vlv_s0ix_state { 23 /* GAM */ 24 u32 wr_watermark; 25 u32 gfx_prio_ctrl; 26 u32 arb_mode; 27 u32 gfx_pend_tlb0; 28 u32 gfx_pend_tlb1; 29 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; 30 u32 media_max_req_count; 31 u32 gfx_max_req_count; 32 u32 render_hwsp; 33 u32 ecochk; 34 u32 bsd_hwsp; 35 u32 blt_hwsp; 36 u32 tlb_rd_addr; 37 38 /* MBC */ 39 u32 g3dctl; 40 u32 gsckgctl; 41 u32 mbctl; 42 43 /* GCP */ 44 u32 ucgctl1; 45 u32 ucgctl3; 46 u32 rcgctl1; 47 u32 rcgctl2; 48 u32 rstctl; 49 u32 misccpctl; 50 51 /* GPM */ 52 u32 gfxpause; 53 u32 rpdeuhwtc; 54 u32 rpdeuc; 55 u32 ecobus; 56 u32 pwrdwnupctl; 57 u32 rp_down_timeout; 58 u32 rp_deucsw; 59 u32 rcubmabdtmr; 60 u32 rcedata; 61 u32 spare2gh; 62 63 /* Display 1 CZ domain */ 64 u32 gt_imr; 65 u32 gt_ier; 66 u32 pm_imr; 67 u32 pm_ier; 68 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; 69 70 /* GT SA CZ domain */ 71 u32 tilectl; 72 u32 gt_fifoctl; 73 u32 gtlc_wake_ctrl; 74 u32 gtlc_survive; 75 u32 pmwgicz; 76 77 /* Display 2 CZ domain */ 78 u32 gu_ctl0; 79 u32 gu_ctl1; 80 u32 pcbr; 81 u32 clock_gate_dis2; 82 }; 83 84 /* 85 * Save all Gunit registers that may be lost after a D3 and a subsequent 86 * S0i[R123] transition. The list of registers needing a save/restore is 87 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit 88 * registers in the following way: 89 * - Driver: saved/restored by the driver 90 * - Punit : saved/restored by the Punit firmware 91 * - No, w/o marking: no need to save/restore, since the register is R/O or 92 * used internally by the HW in a way that doesn't depend 93 * keeping the content across a suspend/resume. 94 * - Debug : used for debugging 95 * 96 * We save/restore all registers marked with 'Driver', with the following 97 * exceptions: 98 * - Registers out of use, including also registers marked with 'Debug'. 99 * These have no effect on the driver's operation, so we don't save/restore 100 * them to reduce the overhead. 101 * - Registers that are fully setup by an initialization function called from 102 * the resume path. For example many clock gating and RPS/RC6 registers. 103 * - Registers that provide the right functionality with their reset defaults. 104 * 105 * TODO: Except for registers that based on the above 3 criteria can be safely 106 * ignored, we save/restore all others, practically treating the HW context as 107 * a black-box for the driver. Further investigation is needed to reduce the 108 * saved/restored registers even further, by following the same 3 criteria. 109 */ 110 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915) 111 { 112 struct vlv_s0ix_state *s = i915->vlv_s0ix_state; 113 struct intel_uncore *uncore = &i915->uncore; 114 int i; 115 116 if (!s) 117 return; 118 119 /* GAM 0x4000-0x4770 */ 120 s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK); 121 s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL); 122 s->arb_mode = intel_uncore_read(uncore, ARB_MODE); 123 s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0); 124 s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1); 125 126 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 127 s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i)); 128 129 s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT); 130 s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT); 131 132 s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7); 133 s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK); 134 s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7); 135 s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7); 136 137 s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR); 138 139 /* MBC 0x9024-0x91D0, 0x8500 */ 140 s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL); 141 s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL); 142 s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL); 143 144 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 145 s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1); 146 s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3); 147 s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1); 148 s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2); 149 s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL); 150 s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL); 151 152 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 153 s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE); 154 s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC); 155 s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC); 156 s->ecobus = intel_uncore_read(uncore, ECOBUS); 157 s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL); 158 s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT); 159 s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW); 160 s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR); 161 s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA); 162 s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H); 163 164 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 165 s->gt_imr = intel_uncore_read(uncore, GTIMR); 166 s->gt_ier = intel_uncore_read(uncore, GTIER); 167 s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 168 s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 169 170 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 171 s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i)); 172 173 /* GT SA CZ domain, 0x100000-0x138124 */ 174 s->tilectl = intel_uncore_read(uncore, TILECTL); 175 s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL); 176 s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); 177 s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); 178 s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ); 179 180 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 181 s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0); 182 s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1); 183 s->pcbr = intel_uncore_read(uncore, VLV_PCBR); 184 s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2); 185 186 /* 187 * Not saving any of: 188 * DFT, 0x9800-0x9EC0 189 * SARB, 0xB000-0xB1FC 190 * GAC, 0x5208-0x524C, 0x14000-0x14C000 191 * PCI CFG 192 */ 193 } 194 195 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915) 196 { 197 struct vlv_s0ix_state *s = i915->vlv_s0ix_state; 198 struct intel_uncore *uncore = &i915->uncore; 199 int i; 200 201 if (!s) 202 return; 203 204 /* GAM 0x4000-0x4770 */ 205 intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark); 206 intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); 207 intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16)); 208 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); 209 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); 210 211 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 212 intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]); 213 214 intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); 215 intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); 216 217 intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp); 218 intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk); 219 intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp); 220 intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp); 221 222 intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr); 223 224 /* MBC 0x9024-0x91D0, 0x8500 */ 225 intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl); 226 intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl); 227 intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl); 228 229 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 230 intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1); 231 intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3); 232 intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1); 233 intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2); 234 intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl); 235 intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl); 236 237 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 238 intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause); 239 intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc); 240 intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc); 241 intel_uncore_write(uncore, ECOBUS, s->ecobus); 242 intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl); 243 intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout); 244 intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw); 245 intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr); 246 intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata); 247 intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh); 248 249 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 250 intel_uncore_write(uncore, GTIMR, s->gt_imr); 251 intel_uncore_write(uncore, GTIER, s->gt_ier); 252 intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr); 253 intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier); 254 255 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 256 intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]); 257 258 /* GT SA CZ domain, 0x100000-0x138124 */ 259 intel_uncore_write(uncore, TILECTL, s->tilectl); 260 intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl); 261 /* 262 * Preserve the GT allow wake and GFX force clock bit, they are not 263 * be restored, as they are used to control the s0ix suspend/resume 264 * sequence by the caller. 265 */ 266 intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, ~VLV_GTLC_ALLOWWAKEREQ, 267 s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ); 268 269 intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, ~VLV_GFX_CLK_FORCE_ON_BIT, 270 s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT); 271 272 intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz); 273 274 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 275 intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0); 276 intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1); 277 intel_uncore_write(uncore, VLV_PCBR, s->pcbr); 278 intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); 279 } 280 281 static int vlv_wait_for_pw_status(struct drm_i915_private *i915, 282 u32 mask, u32 val) 283 { 284 i915_reg_t reg = VLV_GTLC_PW_STATUS; 285 u32 reg_value; 286 int ret; 287 288 /* The HW does not like us polling for PW_STATUS frequently, so 289 * use the sleeping loop rather than risk the busy spin within 290 * intel_wait_for_register(). 291 * 292 * Transitioning between RC6 states should be at most 2ms (see 293 * valleyview_enable_rps) so use a 3ms timeout. 294 */ 295 ret = wait_for(((reg_value = 296 intel_uncore_read_notrace(&i915->uncore, reg)) & mask) 297 == val, 3); 298 299 /* just trace the final value */ 300 trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); 301 302 return ret; 303 } 304 305 static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on) 306 { 307 struct intel_uncore *uncore = &i915->uncore; 308 int err; 309 310 intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, VLV_GFX_CLK_FORCE_ON_BIT, 311 force_on ? VLV_GFX_CLK_FORCE_ON_BIT : 0); 312 313 if (!force_on) 314 return 0; 315 316 err = intel_wait_for_register(uncore, 317 VLV_GTLC_SURVIVABILITY_REG, 318 VLV_GFX_CLK_STATUS_BIT, 319 VLV_GFX_CLK_STATUS_BIT, 320 20); 321 if (err) 322 drm_err(&i915->drm, 323 "timeout waiting for GFX clock force-on (%08x)\n", 324 intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG)); 325 326 return err; 327 } 328 329 static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow) 330 { 331 struct intel_uncore *uncore = &i915->uncore; 332 u32 mask; 333 u32 val; 334 int err; 335 336 intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ, 337 allow ? VLV_GTLC_ALLOWWAKEREQ : 0); 338 intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL); 339 340 mask = VLV_GTLC_ALLOWWAKEACK; 341 val = allow ? mask : 0; 342 343 err = vlv_wait_for_pw_status(i915, mask, val); 344 if (err) 345 drm_err(&i915->drm, "timeout disabling GT waking\n"); 346 347 return err; 348 } 349 350 static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, 351 bool wait_for_on) 352 { 353 u32 mask; 354 u32 val; 355 356 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; 357 val = wait_for_on ? mask : 0; 358 359 /* 360 * RC6 transitioning can be delayed up to 2 msec (see 361 * valleyview_enable_rps), use 3 msec for safety. 362 * 363 * This can fail to turn off the rc6 if the GPU is stuck after a failed 364 * reset and we are trying to force the machine to sleep. 365 */ 366 if (vlv_wait_for_pw_status(dev_priv, mask, val)) 367 drm_dbg(&dev_priv->drm, 368 "timeout waiting for GT wells to go %s\n", 369 str_on_off(wait_for_on)); 370 } 371 372 static void vlv_check_no_gt_access(struct drm_i915_private *i915) 373 { 374 struct intel_uncore *uncore = &i915->uncore; 375 376 if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) 377 return; 378 379 drm_dbg(&i915->drm, "GT register access while GT waking disabled\n"); 380 intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); 381 } 382 383 int vlv_suspend_complete(struct drm_i915_private *dev_priv) 384 { 385 u32 mask; 386 int err; 387 388 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 389 return 0; 390 391 /* 392 * Bspec defines the following GT well on flags as debug only, so 393 * don't treat them as hard failures. 394 */ 395 vlv_wait_for_gt_wells(dev_priv, false); 396 397 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; 398 drm_WARN_ON(&dev_priv->drm, 399 (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask); 400 401 vlv_check_no_gt_access(dev_priv); 402 403 err = vlv_force_gfx_clock(dev_priv, true); 404 if (err) 405 goto err1; 406 407 err = vlv_allow_gt_wake(dev_priv, false); 408 if (err) 409 goto err2; 410 411 vlv_save_gunit_s0ix_state(dev_priv); 412 413 err = vlv_force_gfx_clock(dev_priv, false); 414 if (err) 415 goto err2; 416 417 return 0; 418 419 err2: 420 /* For safety always re-enable waking and disable gfx clock forcing */ 421 vlv_allow_gt_wake(dev_priv, true); 422 err1: 423 vlv_force_gfx_clock(dev_priv, false); 424 425 return err; 426 } 427 428 int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) 429 { 430 int err; 431 int ret; 432 433 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 434 return 0; 435 436 /* 437 * If any of the steps fail just try to continue, that's the best we 438 * can do at this point. Return the first error code (which will also 439 * leave RPM permanently disabled). 440 */ 441 ret = vlv_force_gfx_clock(dev_priv, true); 442 443 vlv_restore_gunit_s0ix_state(dev_priv); 444 445 err = vlv_allow_gt_wake(dev_priv, true); 446 if (!ret) 447 ret = err; 448 449 err = vlv_force_gfx_clock(dev_priv, false); 450 if (!ret) 451 ret = err; 452 453 vlv_check_no_gt_access(dev_priv); 454 455 if (rpm_resume) 456 intel_clock_gating_init(dev_priv); 457 458 return ret; 459 } 460 461 int vlv_suspend_init(struct drm_i915_private *i915) 462 { 463 if (!IS_VALLEYVIEW(i915)) 464 return 0; 465 466 /* we write all the values in the struct, so no need to zero it out */ 467 i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state), 468 GFP_KERNEL); 469 if (!i915->vlv_s0ix_state) 470 return -ENOMEM; 471 472 return 0; 473 } 474 475 void vlv_suspend_cleanup(struct drm_i915_private *i915) 476 { 477 if (!i915->vlv_s0ix_state) 478 return; 479 480 kfree(i915->vlv_s0ix_state); 481 i915->vlv_s0ix_state = NULL; 482 } 483