1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include <generated/xe_wa_oob.h> 9 #include "xe_force_wake.h" 10 #include "xe_device.h" 11 #include "xe_gt.h" 12 #include "xe_gt_idle.h" 13 #include "xe_gt_sysfs.h" 14 #include "xe_guc_pc.h" 15 #include "regs/xe_gt_regs.h" 16 #include "xe_macros.h" 17 #include "xe_mmio.h" 18 #include "xe_pm.h" 19 #include "xe_sriov.h" 20 #include "xe_wa.h" 21 22 /** 23 * DOC: Xe GT Idle 24 * 25 * Contains functions that init GT idle features like C6 26 * 27 * device/gt#/gtidle/name - name of the state 28 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms 29 * device/gt#/gtidle/idle_status - Provides current idle state 30 */ 31 32 static struct xe_gt_idle *dev_to_gtidle(struct device *dev) 33 { 34 struct kobject *kobj = &dev->kobj; 35 36 return &kobj_to_gt(kobj->parent)->gtidle; 37 } 38 39 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle) 40 { 41 return container_of(gtidle, struct xe_gt, gtidle); 42 } 43 44 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle) 45 { 46 return >idle_to_gt(gtidle)->uc.guc.pc; 47 } 48 49 static struct xe_device * 50 pc_to_xe(struct xe_guc_pc *pc) 51 { 52 struct xe_guc *guc = container_of(pc, struct xe_guc, pc); 53 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc); 54 55 return gt_to_xe(gt); 56 } 57 58 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state) 59 { 60 switch (state) { 61 case GT_IDLE_C0: 62 return "gt-c0"; 63 case GT_IDLE_C6: 64 return "gt-c6"; 65 default: 66 return "unknown"; 67 } 68 } 69 70 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) 71 { 72 u64 delta, overflow_residency, prev_residency; 73 74 lockdep_assert_held(>idle->lock); 75 76 overflow_residency = BIT_ULL(32); 77 78 /* 79 * Counter wrap handling 80 * Store previous hw counter values for counter wrap-around handling 81 * Relying on sufficient frequency of queries otherwise counters can still wrap. 82 */ 83 prev_residency = gtidle->prev_residency; 84 gtidle->prev_residency = cur_residency; 85 86 /* delta */ 87 if (cur_residency >= prev_residency) 88 delta = cur_residency - prev_residency; 89 else 90 delta = cur_residency + (overflow_residency - prev_residency); 91 92 /* Add delta to extended raw driver copy of idle residency */ 93 cur_residency = gtidle->cur_residency + delta; 94 gtidle->cur_residency = cur_residency; 95 96 /* residency multiplier in ns, convert to ms */ 97 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6); 98 99 return cur_residency; 100 } 101 102 void xe_gt_idle_enable_pg(struct xe_gt *gt) 103 { 104 struct xe_device *xe = gt_to_xe(gt); 105 struct xe_gt_idle *gtidle = >->gtidle; 106 struct xe_mmio *mmio = >->mmio; 107 u32 vcs_mask, vecs_mask; 108 unsigned int fw_ref; 109 int i, j; 110 111 if (IS_SRIOV_VF(xe)) 112 return; 113 114 /* Disable CPG for PVC */ 115 if (xe->info.platform == XE_PVC) 116 return; 117 118 xe_device_assert_mem_access(gt_to_xe(gt)); 119 120 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 121 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 122 123 if (vcs_mask || vecs_mask) 124 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE; 125 126 if (xe_gt_is_main_type(gt)) 127 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE; 128 129 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 130 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE; 131 132 if (xe->info.platform != XE_DG1) { 133 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { 134 if ((gt->info.engine_mask & BIT(i))) 135 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) | 136 VDN_MFXVDENC_POWERGATE_ENABLE(j)); 137 } 138 } 139 140 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 141 if (xe->info.skip_guc_pc) { 142 /* 143 * GuC sets the hysteresis value when GuC PC is enabled 144 * else set it to 25 (25 * 1.28us) 145 */ 146 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25); 147 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); 148 } 149 150 if (XE_GT_WA(gt, 14020316580)) 151 gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) | 152 VDN_MFXVDENC_POWERGATE_ENABLE(0) | 153 VDN_HCP_POWERGATE_ENABLE(2) | 154 VDN_MFXVDENC_POWERGATE_ENABLE(2)); 155 156 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 157 xe_force_wake_put(gt_to_fw(gt), fw_ref); 158 } 159 160 void xe_gt_idle_disable_pg(struct xe_gt *gt) 161 { 162 struct xe_gt_idle *gtidle = >->gtidle; 163 unsigned int fw_ref; 164 165 if (IS_SRIOV_VF(gt_to_xe(gt))) 166 return; 167 168 xe_device_assert_mem_access(gt_to_xe(gt)); 169 gtidle->powergate_enable = 0; 170 171 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 172 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 173 xe_force_wake_put(gt_to_fw(gt), fw_ref); 174 } 175 176 /** 177 * xe_gt_idle_pg_print - Xe powergating info 178 * @gt: GT object 179 * @p: drm_printer. 180 * 181 * This function prints the powergating information 182 * 183 * Return: 0 on success, negative error code otherwise 184 */ 185 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) 186 { 187 struct xe_gt_idle *gtidle = >->gtidle; 188 struct xe_device *xe = gt_to_xe(gt); 189 enum xe_gt_idle_state state; 190 u32 pg_enabled, pg_status = 0; 191 u32 vcs_mask, vecs_mask; 192 unsigned int fw_ref; 193 int n; 194 /* 195 * Media Slices 196 * 197 * Slice 0: VCS0, VCS1, VECS0 198 * Slice 1: VCS2, VCS3, VECS1 199 * Slice 2: VCS4, VCS5, VECS2 200 * Slice 3: VCS6, VCS7, VECS3 201 */ 202 static const struct { 203 u64 engines; 204 u32 status_bit; 205 } media_slices[] = { 206 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) | 207 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS}, 208 209 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) | 210 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS}, 211 212 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) | 213 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS}, 214 215 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) | 216 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS}, 217 }; 218 219 if (xe->info.platform == XE_PVC) { 220 drm_printf(p, "Power Gating not supported\n"); 221 return 0; 222 } 223 224 state = gtidle->idle_status(gtidle_to_pc(gtidle)); 225 pg_enabled = gtidle->powergate_enable; 226 227 /* Do not wake the GT to read powergating status */ 228 if (state != GT_IDLE_C6) { 229 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 230 if (!fw_ref) 231 return -ETIMEDOUT; 232 233 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE); 234 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); 235 236 xe_force_wake_put(gt_to_fw(gt), fw_ref); 237 } 238 239 if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { 240 drm_printf(p, "Render Power Gating Enabled: %s\n", 241 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); 242 243 drm_printf(p, "Render Power Gate Status: %s\n", 244 str_up_down(pg_status & RENDER_AWAKE_STATUS)); 245 } 246 247 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 248 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 249 250 /* Print media CPG status only if media is present */ 251 if (vcs_mask || vecs_mask) { 252 drm_printf(p, "Media Power Gating Enabled: %s\n", 253 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE)); 254 255 for (n = 0; n < ARRAY_SIZE(media_slices); n++) 256 if (gt->info.engine_mask & media_slices[n].engines) 257 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n, 258 str_up_down(pg_status & media_slices[n].status_bit)); 259 } 260 261 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 262 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n", 263 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE)); 264 265 return 0; 266 } 267 268 static ssize_t name_show(struct kobject *kobj, 269 struct kobj_attribute *attr, char *buff) 270 { 271 struct device *dev = kobj_to_dev(kobj); 272 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 273 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 274 ssize_t ret; 275 276 xe_pm_runtime_get(pc_to_xe(pc)); 277 ret = sysfs_emit(buff, "%s\n", gtidle->name); 278 xe_pm_runtime_put(pc_to_xe(pc)); 279 280 return ret; 281 } 282 static struct kobj_attribute name_attr = __ATTR_RO(name); 283 284 static ssize_t idle_status_show(struct kobject *kobj, 285 struct kobj_attribute *attr, char *buff) 286 { 287 struct device *dev = kobj_to_dev(kobj); 288 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 289 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 290 enum xe_gt_idle_state state; 291 292 xe_pm_runtime_get(pc_to_xe(pc)); 293 state = gtidle->idle_status(pc); 294 xe_pm_runtime_put(pc_to_xe(pc)); 295 296 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state)); 297 } 298 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status); 299 300 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle) 301 { 302 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 303 u64 residency; 304 unsigned long flags; 305 306 raw_spin_lock_irqsave(>idle->lock, flags); 307 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc)); 308 raw_spin_unlock_irqrestore(>idle->lock, flags); 309 310 return residency; 311 } 312 313 314 static ssize_t idle_residency_ms_show(struct kobject *kobj, 315 struct kobj_attribute *attr, char *buff) 316 { 317 struct device *dev = kobj_to_dev(kobj); 318 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 319 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 320 u64 residency; 321 322 xe_pm_runtime_get(pc_to_xe(pc)); 323 residency = xe_gt_idle_residency_msec(gtidle); 324 xe_pm_runtime_put(pc_to_xe(pc)); 325 326 return sysfs_emit(buff, "%llu\n", residency); 327 } 328 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms); 329 330 static const struct attribute *gt_idle_attrs[] = { 331 &name_attr.attr, 332 &idle_status_attr.attr, 333 &idle_residency_attr.attr, 334 NULL, 335 }; 336 337 static void gt_idle_fini(void *arg) 338 { 339 struct kobject *kobj = arg; 340 struct xe_gt *gt = kobj_to_gt(kobj->parent); 341 342 xe_gt_idle_disable_pg(gt); 343 344 if (gt_to_xe(gt)->info.skip_guc_pc) 345 xe_gt_idle_disable_c6(gt); 346 347 sysfs_remove_files(kobj, gt_idle_attrs); 348 kobject_put(kobj); 349 } 350 351 int xe_gt_idle_init(struct xe_gt_idle *gtidle) 352 { 353 struct xe_gt *gt = gtidle_to_gt(gtidle); 354 struct xe_device *xe = gt_to_xe(gt); 355 struct kobject *kobj; 356 int err; 357 358 if (IS_SRIOV_VF(xe)) 359 return 0; 360 361 kobj = kobject_create_and_add("gtidle", gt->sysfs); 362 if (!kobj) 363 return -ENOMEM; 364 365 raw_spin_lock_init(>idle->lock); 366 367 if (xe_gt_is_media_type(gt)) { 368 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id); 369 gtidle->idle_residency = xe_guc_pc_mc6_residency; 370 } else { 371 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id); 372 gtidle->idle_residency = xe_guc_pc_rc6_residency; 373 } 374 375 /* Multiplier for Residency counter in units of 1.28us */ 376 gtidle->residency_multiplier = 1280; 377 gtidle->idle_status = xe_guc_pc_c_status; 378 379 err = sysfs_create_files(kobj, gt_idle_attrs); 380 if (err) { 381 kobject_put(kobj); 382 return err; 383 } 384 385 xe_gt_idle_enable_pg(gt); 386 387 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj); 388 } 389 390 void xe_gt_idle_enable_c6(struct xe_gt *gt) 391 { 392 xe_device_assert_mem_access(gt_to_xe(gt)); 393 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 394 395 if (IS_SRIOV_VF(gt_to_xe(gt))) 396 return; 397 398 /* Units of 1280 ns for a total of 5s */ 399 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA); 400 /* Enable RC6 */ 401 xe_mmio_write32(>->mmio, RC_CONTROL, 402 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE); 403 } 404 405 int xe_gt_idle_disable_c6(struct xe_gt *gt) 406 { 407 unsigned int fw_ref; 408 409 xe_device_assert_mem_access(gt_to_xe(gt)); 410 411 if (IS_SRIOV_VF(gt_to_xe(gt))) 412 return 0; 413 414 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 415 if (!fw_ref) 416 return -ETIMEDOUT; 417 418 xe_mmio_write32(>->mmio, RC_CONTROL, 0); 419 xe_mmio_write32(>->mmio, RC_STATE, 0); 420 421 xe_force_wake_put(gt_to_fw(gt), fw_ref); 422 423 return 0; 424 } 425