1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include <generated/xe_wa_oob.h> 9 #include "xe_force_wake.h" 10 #include "xe_device.h" 11 #include "xe_gt.h" 12 #include "xe_gt_idle.h" 13 #include "xe_gt_sysfs.h" 14 #include "xe_guc_pc.h" 15 #include "regs/xe_gt_regs.h" 16 #include "xe_mmio.h" 17 #include "xe_pm.h" 18 #include "xe_sriov.h" 19 #include "xe_wa.h" 20 21 /** 22 * DOC: Xe GT Idle 23 * 24 * Contains functions that init GT idle features like C6 25 * 26 * device/gt#/gtidle/name - name of the state 27 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms 28 * device/gt#/gtidle/idle_status - Provides current idle state 29 */ 30 31 static struct xe_gt_idle *dev_to_gtidle(struct device *dev) 32 { 33 struct kobject *kobj = &dev->kobj; 34 35 return &kobj_to_gt(kobj->parent)->gtidle; 36 } 37 38 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle) 39 { 40 return container_of(gtidle, struct xe_gt, gtidle); 41 } 42 43 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle) 44 { 45 return >idle_to_gt(gtidle)->uc.guc.pc; 46 } 47 48 static struct xe_device * 49 pc_to_xe(struct xe_guc_pc *pc) 50 { 51 struct xe_guc *guc = container_of(pc, struct xe_guc, pc); 52 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc); 53 54 return gt_to_xe(gt); 55 } 56 57 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state) 58 { 59 switch (state) { 60 case GT_IDLE_C0: 61 return "gt-c0"; 62 case GT_IDLE_C6: 63 return "gt-c6"; 64 default: 65 return "unknown"; 66 } 67 } 68 69 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) 70 { 71 u64 delta, overflow_residency, prev_residency; 72 73 lockdep_assert_held(>idle->lock); 74 75 overflow_residency = BIT_ULL(32); 76 77 /* 78 * Counter wrap handling 79 * Store previous hw counter values for counter wrap-around handling 80 * Relying on sufficient frequency of queries otherwise counters can still wrap. 81 */ 82 prev_residency = gtidle->prev_residency; 83 gtidle->prev_residency = cur_residency; 84 85 /* delta */ 86 if (cur_residency >= prev_residency) 87 delta = cur_residency - prev_residency; 88 else 89 delta = cur_residency + (overflow_residency - prev_residency); 90 91 /* Add delta to extended raw driver copy of idle residency */ 92 cur_residency = gtidle->cur_residency + delta; 93 gtidle->cur_residency = cur_residency; 94 95 /* residency multiplier in ns, convert to ms */ 96 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6); 97 98 return cur_residency; 99 } 100 101 void xe_gt_idle_enable_pg(struct xe_gt *gt) 102 { 103 struct xe_device *xe = gt_to_xe(gt); 104 struct xe_gt_idle *gtidle = >->gtidle; 105 struct xe_mmio *mmio = >->mmio; 106 u32 vcs_mask, vecs_mask; 107 int i, j; 108 109 if (IS_SRIOV_VF(xe)) 110 return; 111 112 /* Disable CPG for PVC */ 113 if (xe->info.platform == XE_PVC) 114 return; 115 116 xe_device_assert_mem_access(gt_to_xe(gt)); 117 118 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 119 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 120 121 if (vcs_mask || vecs_mask) 122 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE; 123 124 if (xe_gt_is_main_type(gt)) 125 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE; 126 127 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 128 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE; 129 130 if (xe->info.platform != XE_DG1) { 131 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { 132 if ((gt->info.engine_mask & BIT(i))) 133 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) | 134 VDN_MFXVDENC_POWERGATE_ENABLE(j)); 135 } 136 } 137 138 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 139 if (xe->info.skip_guc_pc) { 140 /* 141 * GuC sets the hysteresis value when GuC PC is enabled 142 * else set it to 25 (25 * 1.28us) 143 */ 144 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25); 145 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); 146 } 147 148 if (XE_GT_WA(gt, 14020316580)) 149 gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) | 150 VDN_MFXVDENC_POWERGATE_ENABLE(0) | 151 VDN_HCP_POWERGATE_ENABLE(2) | 152 VDN_MFXVDENC_POWERGATE_ENABLE(2)); 153 154 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 155 } 156 157 void xe_gt_idle_disable_pg(struct xe_gt *gt) 158 { 159 struct xe_gt_idle *gtidle = >->gtidle; 160 161 if (IS_SRIOV_VF(gt_to_xe(gt))) 162 return; 163 164 xe_device_assert_mem_access(gt_to_xe(gt)); 165 gtidle->powergate_enable = 0; 166 167 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 168 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 169 } 170 171 static void force_wake_domains_show(struct xe_gt *gt, struct drm_printer *p) 172 { 173 struct xe_force_wake_domain *domain; 174 struct xe_force_wake *fw = gt_to_fw(gt); 175 unsigned int tmp; 176 unsigned long flags; 177 178 spin_lock_irqsave(&fw->lock, flags); 179 for_each_fw_domain(domain, fw, tmp) { 180 drm_printf(p, "%s.ref_count=%u, %s.fwake=0x%x\n", 181 xe_force_wake_domain_to_str(domain->id), 182 READ_ONCE(domain->ref), 183 xe_force_wake_domain_to_str(domain->id), 184 xe_mmio_read32(>->mmio, domain->reg_ctl)); 185 } 186 spin_unlock_irqrestore(&fw->lock, flags); 187 } 188 189 /** 190 * xe_gt_idle_pg_print - Xe powergating info 191 * @gt: GT object 192 * @p: drm_printer. 193 * 194 * This function prints the powergating information 195 * 196 * Return: 0 on success, negative error code otherwise 197 */ 198 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) 199 { 200 struct xe_gt_idle *gtidle = >->gtidle; 201 struct xe_device *xe = gt_to_xe(gt); 202 enum xe_gt_idle_state state; 203 u32 pg_enabled, pg_status = 0; 204 u32 vcs_mask, vecs_mask; 205 int n; 206 /* 207 * Media Slices 208 * 209 * Slice 0: VCS0, VCS1, VECS0 210 * Slice 1: VCS2, VCS3, VECS1 211 * Slice 2: VCS4, VCS5, VECS2 212 * Slice 3: VCS6, VCS7, VECS3 213 */ 214 static const struct { 215 u64 engines; 216 u32 status_bit; 217 } media_slices[] = { 218 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) | 219 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS}, 220 221 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) | 222 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS}, 223 224 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) | 225 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS}, 226 227 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) | 228 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS}, 229 }; 230 231 if (xe->info.platform == XE_PVC) { 232 drm_printf(p, "Power Gating not supported\n"); 233 return 0; 234 } 235 236 state = gtidle->idle_status(gtidle_to_pc(gtidle)); 237 pg_enabled = gtidle->powergate_enable; 238 239 /* Do not wake the GT to read powergating status */ 240 if (state != GT_IDLE_C6) { 241 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 242 if (!fw_ref.domains) 243 return -ETIMEDOUT; 244 245 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE); 246 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); 247 } 248 249 if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { 250 drm_printf(p, "Render Power Gating Enabled: %s\n", 251 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); 252 253 drm_printf(p, "Render Power Gate Status: %s\n", 254 str_up_down(pg_status & RENDER_AWAKE_STATUS)); 255 } 256 257 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 258 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 259 260 /* Print media CPG status only if media is present */ 261 if (vcs_mask || vecs_mask) { 262 drm_printf(p, "Media Power Gating Enabled: %s\n", 263 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE)); 264 265 for (n = 0; n < ARRAY_SIZE(media_slices); n++) 266 if (gt->info.engine_mask & media_slices[n].engines) 267 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n, 268 str_up_down(pg_status & media_slices[n].status_bit)); 269 } 270 271 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 272 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n", 273 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE)); 274 275 if (gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)) { 276 drm_printf(p, "GSC Power Gate Status: %s\n", 277 str_up_down(pg_status & GSC_AWAKE_STATUS)); 278 } 279 280 force_wake_domains_show(gt, p); 281 282 return 0; 283 } 284 285 static ssize_t name_show(struct kobject *kobj, 286 struct kobj_attribute *attr, char *buff) 287 { 288 struct device *dev = kobj_to_dev(kobj); 289 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 290 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 291 292 guard(xe_pm_runtime)(pc_to_xe(pc)); 293 return sysfs_emit(buff, "%s\n", gtidle->name); 294 } 295 static struct kobj_attribute name_attr = __ATTR_RO(name); 296 297 static ssize_t idle_status_show(struct kobject *kobj, 298 struct kobj_attribute *attr, char *buff) 299 { 300 struct device *dev = kobj_to_dev(kobj); 301 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 302 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 303 enum xe_gt_idle_state state; 304 305 scoped_guard(xe_pm_runtime, pc_to_xe(pc)) 306 state = gtidle->idle_status(pc); 307 308 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state)); 309 } 310 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status); 311 312 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle) 313 { 314 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 315 u64 residency; 316 unsigned long flags; 317 318 raw_spin_lock_irqsave(>idle->lock, flags); 319 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc)); 320 raw_spin_unlock_irqrestore(>idle->lock, flags); 321 322 return residency; 323 } 324 325 326 static ssize_t idle_residency_ms_show(struct kobject *kobj, 327 struct kobj_attribute *attr, char *buff) 328 { 329 struct device *dev = kobj_to_dev(kobj); 330 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 331 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 332 u64 residency; 333 334 scoped_guard(xe_pm_runtime, pc_to_xe(pc)) 335 residency = xe_gt_idle_residency_msec(gtidle); 336 337 return sysfs_emit(buff, "%llu\n", residency); 338 } 339 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms); 340 341 static const struct attribute *gt_idle_attrs[] = { 342 &name_attr.attr, 343 &idle_status_attr.attr, 344 &idle_residency_attr.attr, 345 NULL, 346 }; 347 348 static void gt_idle_fini(void *arg) 349 { 350 struct kobject *kobj = arg; 351 struct xe_gt *gt = kobj_to_gt(kobj->parent); 352 353 xe_gt_idle_disable_pg(gt); 354 355 if (gt_to_xe(gt)->info.skip_guc_pc) 356 xe_gt_idle_disable_c6(gt); 357 358 sysfs_remove_files(kobj, gt_idle_attrs); 359 kobject_put(kobj); 360 } 361 362 int xe_gt_idle_init(struct xe_gt_idle *gtidle) 363 { 364 struct xe_gt *gt = gtidle_to_gt(gtidle); 365 struct xe_device *xe = gt_to_xe(gt); 366 struct kobject *kobj; 367 int err; 368 369 if (IS_SRIOV_VF(xe)) 370 return 0; 371 372 kobj = kobject_create_and_add("gtidle", gt->sysfs); 373 if (!kobj) 374 return -ENOMEM; 375 376 raw_spin_lock_init(>idle->lock); 377 378 if (xe_gt_is_media_type(gt)) { 379 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id); 380 gtidle->idle_residency = xe_guc_pc_mc6_residency; 381 } else { 382 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id); 383 gtidle->idle_residency = xe_guc_pc_rc6_residency; 384 } 385 386 /* Multiplier for Residency counter in units of 1.28us */ 387 gtidle->residency_multiplier = 1280; 388 gtidle->idle_status = xe_guc_pc_c_status; 389 390 err = sysfs_create_files(kobj, gt_idle_attrs); 391 if (err) { 392 kobject_put(kobj); 393 return err; 394 } 395 396 xe_gt_idle_enable_pg(gt); 397 398 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj); 399 } 400 401 void xe_gt_idle_enable_c6(struct xe_gt *gt) 402 { 403 xe_device_assert_mem_access(gt_to_xe(gt)); 404 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 405 406 if (IS_SRIOV_VF(gt_to_xe(gt))) 407 return; 408 409 /* Units of 1280 ns for a total of 5s */ 410 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA); 411 /* Enable RC6 */ 412 xe_mmio_write32(>->mmio, RC_CONTROL, 413 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE); 414 } 415 416 int xe_gt_idle_disable_c6(struct xe_gt *gt) 417 { 418 xe_device_assert_mem_access(gt_to_xe(gt)); 419 420 if (IS_SRIOV_VF(gt_to_xe(gt))) 421 return 0; 422 423 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 424 if (!fw_ref.domains) 425 return -ETIMEDOUT; 426 427 xe_mmio_write32(>->mmio, RC_CONTROL, 0); 428 xe_mmio_write32(>->mmio, RC_STATE, 0); 429 430 return 0; 431 } 432