1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <linux/time64.h> 7 8 #include <drm/drm_managed.h> 9 10 #include <generated/xe_wa_oob.h> 11 #include "xe_force_wake.h" 12 #include "xe_device.h" 13 #include "xe_gt.h" 14 #include "xe_gt_idle.h" 15 #include "xe_gt_sysfs.h" 16 #include "xe_guc_pc.h" 17 #include "regs/xe_gt_regs.h" 18 #include "xe_mmio.h" 19 #include "xe_pm.h" 20 #include "xe_sriov.h" 21 #include "xe_wa.h" 22 23 /** 24 * DOC: Xe GT Idle 25 * 26 * Contains functions that init GT idle features like C6 27 * 28 * device/gt#/gtidle/name - name of the state 29 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms 30 * device/gt#/gtidle/idle_status - Provides current idle state 31 */ 32 33 static struct xe_gt_idle *dev_to_gtidle(struct device *dev) 34 { 35 struct kobject *kobj = &dev->kobj; 36 37 return &kobj_to_gt(kobj->parent)->gtidle; 38 } 39 40 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle) 41 { 42 return container_of(gtidle, struct xe_gt, gtidle); 43 } 44 45 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle) 46 { 47 return >idle_to_gt(gtidle)->uc.guc.pc; 48 } 49 50 static struct xe_device * 51 pc_to_xe(struct xe_guc_pc *pc) 52 { 53 struct xe_guc *guc = container_of(pc, struct xe_guc, pc); 54 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc); 55 56 return gt_to_xe(gt); 57 } 58 59 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state) 60 { 61 switch (state) { 62 case GT_IDLE_C0: 63 return "gt-c0"; 64 case GT_IDLE_C6: 65 return "gt-c6"; 66 default: 67 return "unknown"; 68 } 69 } 70 71 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) 72 { 73 u64 delta, overflow_residency, prev_residency; 74 75 lockdep_assert_held(>idle->lock); 76 77 overflow_residency = BIT_ULL(32); 78 79 /* 80 * Counter wrap handling 81 * Store previous hw counter values for counter wrap-around handling 82 * Relying on sufficient frequency of queries otherwise counters can still wrap. 83 */ 84 prev_residency = gtidle->prev_residency; 85 gtidle->prev_residency = cur_residency; 86 87 /* delta */ 88 if (cur_residency >= prev_residency) 89 delta = cur_residency - prev_residency; 90 else 91 delta = cur_residency + (overflow_residency - prev_residency); 92 93 /* Add delta to extended raw driver copy of idle residency */ 94 cur_residency = gtidle->cur_residency + delta; 95 gtidle->cur_residency = cur_residency; 96 97 /* residency multiplier in ns, convert to ms */ 98 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, NSEC_PER_MSEC); 99 100 return cur_residency; 101 } 102 103 void xe_gt_idle_enable_pg(struct xe_gt *gt) 104 { 105 struct xe_device *xe = gt_to_xe(gt); 106 struct xe_gt_idle *gtidle = >->gtidle; 107 struct xe_mmio *mmio = >->mmio; 108 u32 vcs_mask, vecs_mask; 109 int i, j; 110 111 if (IS_SRIOV_VF(xe)) 112 return; 113 114 /* Disable CPG for PVC */ 115 if (xe->info.platform == XE_PVC) 116 return; 117 118 xe_device_assert_mem_access(gt_to_xe(gt)); 119 120 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 121 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 122 123 if (vcs_mask || vecs_mask) 124 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE; 125 126 if (xe_gt_is_main_type(gt)) 127 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE; 128 129 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 130 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE; 131 132 if (xe->info.platform != XE_DG1) { 133 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { 134 if ((gt->info.engine_mask & BIT(i))) 135 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) | 136 VDN_MFXVDENC_POWERGATE_ENABLE(j)); 137 } 138 } 139 140 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 141 if (xe->info.skip_guc_pc) { 142 /* 143 * GuC sets the hysteresis value when GuC PC is enabled 144 * else set it to 25 (25 * 1.28us) 145 */ 146 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25); 147 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); 148 } 149 150 if (XE_GT_WA(gt, 14020316580)) 151 gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) | 152 VDN_MFXVDENC_POWERGATE_ENABLE(0) | 153 VDN_HCP_POWERGATE_ENABLE(2) | 154 VDN_MFXVDENC_POWERGATE_ENABLE(2)); 155 156 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 157 } 158 159 void xe_gt_idle_disable_pg(struct xe_gt *gt) 160 { 161 struct xe_gt_idle *gtidle = >->gtidle; 162 163 if (IS_SRIOV_VF(gt_to_xe(gt))) 164 return; 165 166 xe_device_assert_mem_access(gt_to_xe(gt)); 167 gtidle->powergate_enable = 0; 168 169 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 170 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 171 } 172 173 static void force_wake_domains_show(struct xe_gt *gt, struct drm_printer *p) 174 { 175 struct xe_force_wake_domain *domain; 176 struct xe_force_wake *fw = gt_to_fw(gt); 177 unsigned int tmp; 178 unsigned long flags; 179 180 spin_lock_irqsave(&fw->lock, flags); 181 for_each_fw_domain(domain, fw, tmp) { 182 drm_printf(p, "%s.ref_count=%u, %s.fwake=0x%x\n", 183 xe_force_wake_domain_to_str(domain->id), 184 READ_ONCE(domain->ref), 185 xe_force_wake_domain_to_str(domain->id), 186 xe_mmio_read32(>->mmio, domain->reg_ctl)); 187 } 188 spin_unlock_irqrestore(&fw->lock, flags); 189 } 190 191 /** 192 * xe_gt_idle_pg_print - Xe powergating info 193 * @gt: GT object 194 * @p: drm_printer. 195 * 196 * This function prints the powergating information 197 * 198 * Return: 0 on success, negative error code otherwise 199 */ 200 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) 201 { 202 struct xe_gt_idle *gtidle = >->gtidle; 203 struct xe_device *xe = gt_to_xe(gt); 204 enum xe_gt_idle_state state; 205 u32 pg_enabled, pg_status = 0; 206 u32 vcs_mask, vecs_mask; 207 int n; 208 /* 209 * Media Slices 210 * 211 * Slice 0: VCS0, VCS1, VECS0 212 * Slice 1: VCS2, VCS3, VECS1 213 * Slice 2: VCS4, VCS5, VECS2 214 * Slice 3: VCS6, VCS7, VECS3 215 */ 216 static const struct { 217 u64 engines; 218 u32 status_bit; 219 } media_slices[] = { 220 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) | 221 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS}, 222 223 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) | 224 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS}, 225 226 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) | 227 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS}, 228 229 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) | 230 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS}, 231 }; 232 233 if (xe->info.platform == XE_PVC) { 234 drm_printf(p, "Power Gating not supported\n"); 235 return 0; 236 } 237 238 state = gtidle->idle_status(gtidle_to_pc(gtidle)); 239 pg_enabled = gtidle->powergate_enable; 240 241 /* Do not wake the GT to read powergating status */ 242 if (state != GT_IDLE_C6) { 243 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 244 if (!fw_ref.domains) 245 return -ETIMEDOUT; 246 247 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE); 248 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); 249 } 250 251 if (gt->info.engine_mask & 252 (XE_HW_ENGINE_RCS_MASK | XE_HW_ENGINE_CCS_MASK)) { 253 drm_printf(p, "Render Power Gating Enabled: %s\n", 254 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); 255 256 drm_printf(p, "Render Power Gate Status: %s\n", 257 str_up_down(pg_status & RENDER_AWAKE_STATUS)); 258 } 259 260 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 261 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 262 263 /* Print media CPG status only if media is present */ 264 if (vcs_mask || vecs_mask) { 265 drm_printf(p, "Media Power Gating Enabled: %s\n", 266 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE)); 267 268 for (n = 0; n < ARRAY_SIZE(media_slices); n++) 269 if (gt->info.engine_mask & media_slices[n].engines) 270 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n, 271 str_up_down(pg_status & media_slices[n].status_bit)); 272 } 273 274 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 275 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n", 276 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE)); 277 278 if (gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)) { 279 drm_printf(p, "GSC Power Gate Status: %s\n", 280 str_up_down(pg_status & GSC_AWAKE_STATUS)); 281 } 282 283 force_wake_domains_show(gt, p); 284 285 return 0; 286 } 287 288 static ssize_t name_show(struct kobject *kobj, 289 struct kobj_attribute *attr, char *buff) 290 { 291 struct device *dev = kobj_to_dev(kobj); 292 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 293 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 294 295 guard(xe_pm_runtime)(pc_to_xe(pc)); 296 return sysfs_emit(buff, "%s\n", gtidle->name); 297 } 298 static struct kobj_attribute name_attr = __ATTR_RO(name); 299 300 static ssize_t idle_status_show(struct kobject *kobj, 301 struct kobj_attribute *attr, char *buff) 302 { 303 struct device *dev = kobj_to_dev(kobj); 304 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 305 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 306 enum xe_gt_idle_state state; 307 308 scoped_guard(xe_pm_runtime, pc_to_xe(pc)) 309 state = gtidle->idle_status(pc); 310 311 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state)); 312 } 313 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status); 314 315 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle) 316 { 317 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 318 u64 residency; 319 unsigned long flags; 320 321 raw_spin_lock_irqsave(>idle->lock, flags); 322 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc)); 323 raw_spin_unlock_irqrestore(>idle->lock, flags); 324 325 return residency; 326 } 327 328 329 static ssize_t idle_residency_ms_show(struct kobject *kobj, 330 struct kobj_attribute *attr, char *buff) 331 { 332 struct device *dev = kobj_to_dev(kobj); 333 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 334 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 335 u64 residency; 336 337 scoped_guard(xe_pm_runtime, pc_to_xe(pc)) 338 residency = xe_gt_idle_residency_msec(gtidle); 339 340 return sysfs_emit(buff, "%llu\n", residency); 341 } 342 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms); 343 344 static const struct attribute *gt_idle_attrs[] = { 345 &name_attr.attr, 346 &idle_status_attr.attr, 347 &idle_residency_attr.attr, 348 NULL, 349 }; 350 351 static void gt_idle_fini(void *arg) 352 { 353 struct kobject *kobj = arg; 354 struct xe_gt *gt = kobj_to_gt(kobj->parent); 355 356 xe_gt_idle_disable_pg(gt); 357 358 if (gt_to_xe(gt)->info.skip_guc_pc) 359 xe_gt_idle_disable_c6(gt); 360 361 sysfs_remove_files(kobj, gt_idle_attrs); 362 kobject_put(kobj); 363 } 364 365 int xe_gt_idle_init(struct xe_gt_idle *gtidle) 366 { 367 struct xe_gt *gt = gtidle_to_gt(gtidle); 368 struct xe_device *xe = gt_to_xe(gt); 369 struct kobject *kobj; 370 int err; 371 372 if (IS_SRIOV_VF(xe)) 373 return 0; 374 375 kobj = kobject_create_and_add("gtidle", gt->sysfs); 376 if (!kobj) 377 return -ENOMEM; 378 379 raw_spin_lock_init(>idle->lock); 380 381 if (xe_gt_is_media_type(gt)) { 382 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id); 383 gtidle->idle_residency = xe_guc_pc_mc6_residency; 384 } else { 385 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id); 386 gtidle->idle_residency = xe_guc_pc_rc6_residency; 387 } 388 389 /* Multiplier for Residency counter in units of 1.28us */ 390 gtidle->residency_multiplier = 1280; 391 gtidle->idle_status = xe_guc_pc_c_status; 392 393 err = sysfs_create_files(kobj, gt_idle_attrs); 394 if (err) { 395 kobject_put(kobj); 396 return err; 397 } 398 399 xe_gt_idle_enable_pg(gt); 400 401 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj); 402 } 403 404 void xe_gt_idle_enable_c6(struct xe_gt *gt) 405 { 406 xe_device_assert_mem_access(gt_to_xe(gt)); 407 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 408 409 if (IS_SRIOV_VF(gt_to_xe(gt))) 410 return; 411 412 /* Units of 1280 ns for a total of 5s */ 413 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA); 414 /* Enable RC6 */ 415 xe_mmio_write32(>->mmio, RC_CONTROL, 416 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE); 417 } 418 419 int xe_gt_idle_disable_c6(struct xe_gt *gt) 420 { 421 xe_device_assert_mem_access(gt_to_xe(gt)); 422 423 if (IS_SRIOV_VF(gt_to_xe(gt))) 424 return 0; 425 426 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 427 if (!fw_ref.domains) 428 return -ETIMEDOUT; 429 430 xe_mmio_write32(>->mmio, RC_CONTROL, 0); 431 xe_mmio_write32(>->mmio, RC_STATE, 0); 432 433 return 0; 434 } 435