1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include "xe_force_wake.h" 9 #include "xe_device.h" 10 #include "xe_gt.h" 11 #include "xe_gt_idle.h" 12 #include "xe_gt_sysfs.h" 13 #include "xe_guc_pc.h" 14 #include "regs/xe_gt_regs.h" 15 #include "xe_macros.h" 16 #include "xe_mmio.h" 17 #include "xe_pm.h" 18 #include "xe_sriov.h" 19 20 /** 21 * DOC: Xe GT Idle 22 * 23 * Contains functions that init GT idle features like C6 24 * 25 * device/gt#/gtidle/name - name of the state 26 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms 27 * device/gt#/gtidle/idle_status - Provides current idle state 28 */ 29 30 static struct xe_gt_idle *dev_to_gtidle(struct device *dev) 31 { 32 struct kobject *kobj = &dev->kobj; 33 34 return &kobj_to_gt(kobj->parent)->gtidle; 35 } 36 37 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle) 38 { 39 return container_of(gtidle, struct xe_gt, gtidle); 40 } 41 42 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle) 43 { 44 return >idle_to_gt(gtidle)->uc.guc.pc; 45 } 46 47 static struct xe_device * 48 pc_to_xe(struct xe_guc_pc *pc) 49 { 50 struct xe_guc *guc = container_of(pc, struct xe_guc, pc); 51 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc); 52 53 return gt_to_xe(gt); 54 } 55 56 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state) 57 { 58 switch (state) { 59 case GT_IDLE_C0: 60 return "gt-c0"; 61 case GT_IDLE_C6: 62 return "gt-c6"; 63 default: 64 return "unknown"; 65 } 66 } 67 68 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) 69 { 70 u64 delta, overflow_residency, prev_residency; 71 72 lockdep_assert_held(>idle->lock); 73 74 overflow_residency = BIT_ULL(32); 75 76 /* 77 * Counter wrap handling 78 * Store previous hw counter values for counter wrap-around handling 79 * Relying on sufficient frequency of queries otherwise counters can still wrap. 80 */ 81 prev_residency = gtidle->prev_residency; 82 gtidle->prev_residency = cur_residency; 83 84 /* delta */ 85 if (cur_residency >= prev_residency) 86 delta = cur_residency - prev_residency; 87 else 88 delta = cur_residency + (overflow_residency - prev_residency); 89 90 /* Add delta to extended raw driver copy of idle residency */ 91 cur_residency = gtidle->cur_residency + delta; 92 gtidle->cur_residency = cur_residency; 93 94 /* residency multiplier in ns, convert to ms */ 95 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6); 96 97 return cur_residency; 98 } 99 100 void xe_gt_idle_enable_pg(struct xe_gt *gt) 101 { 102 struct xe_device *xe = gt_to_xe(gt); 103 struct xe_gt_idle *gtidle = >->gtidle; 104 struct xe_mmio *mmio = >->mmio; 105 u32 vcs_mask, vecs_mask; 106 unsigned int fw_ref; 107 int i, j; 108 109 if (IS_SRIOV_VF(xe)) 110 return; 111 112 /* Disable CPG for PVC */ 113 if (xe->info.platform == XE_PVC) 114 return; 115 116 xe_device_assert_mem_access(gt_to_xe(gt)); 117 118 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 119 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 120 121 if (vcs_mask || vecs_mask) 122 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE; 123 124 if (xe_gt_is_main_type(gt)) 125 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE; 126 127 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 128 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE; 129 130 if (xe->info.platform != XE_DG1) { 131 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { 132 if ((gt->info.engine_mask & BIT(i))) 133 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) | 134 VDN_MFXVDENC_POWERGATE_ENABLE(j)); 135 } 136 } 137 138 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 139 if (xe->info.skip_guc_pc) { 140 /* 141 * GuC sets the hysteresis value when GuC PC is enabled 142 * else set it to 25 (25 * 1.28us) 143 */ 144 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25); 145 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); 146 } 147 148 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 149 xe_force_wake_put(gt_to_fw(gt), fw_ref); 150 } 151 152 void xe_gt_idle_disable_pg(struct xe_gt *gt) 153 { 154 struct xe_gt_idle *gtidle = >->gtidle; 155 unsigned int fw_ref; 156 157 if (IS_SRIOV_VF(gt_to_xe(gt))) 158 return; 159 160 xe_device_assert_mem_access(gt_to_xe(gt)); 161 gtidle->powergate_enable = 0; 162 163 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 164 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable); 165 xe_force_wake_put(gt_to_fw(gt), fw_ref); 166 } 167 168 /** 169 * xe_gt_idle_pg_print - Xe powergating info 170 * @gt: GT object 171 * @p: drm_printer. 172 * 173 * This function prints the powergating information 174 * 175 * Return: 0 on success, negative error code otherwise 176 */ 177 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) 178 { 179 struct xe_gt_idle *gtidle = >->gtidle; 180 struct xe_device *xe = gt_to_xe(gt); 181 enum xe_gt_idle_state state; 182 u32 pg_enabled, pg_status = 0; 183 u32 vcs_mask, vecs_mask; 184 unsigned int fw_ref; 185 int n; 186 /* 187 * Media Slices 188 * 189 * Slice 0: VCS0, VCS1, VECS0 190 * Slice 1: VCS2, VCS3, VECS1 191 * Slice 2: VCS4, VCS5, VECS2 192 * Slice 3: VCS6, VCS7, VECS3 193 */ 194 static const struct { 195 u64 engines; 196 u32 status_bit; 197 } media_slices[] = { 198 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) | 199 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS}, 200 201 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) | 202 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS}, 203 204 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) | 205 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS}, 206 207 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) | 208 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS}, 209 }; 210 211 if (xe->info.platform == XE_PVC) { 212 drm_printf(p, "Power Gating not supported\n"); 213 return 0; 214 } 215 216 state = gtidle->idle_status(gtidle_to_pc(gtidle)); 217 pg_enabled = gtidle->powergate_enable; 218 219 /* Do not wake the GT to read powergating status */ 220 if (state != GT_IDLE_C6) { 221 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 222 if (!fw_ref) 223 return -ETIMEDOUT; 224 225 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE); 226 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); 227 228 xe_force_wake_put(gt_to_fw(gt), fw_ref); 229 } 230 231 if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { 232 drm_printf(p, "Render Power Gating Enabled: %s\n", 233 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); 234 235 drm_printf(p, "Render Power Gate Status: %s\n", 236 str_up_down(pg_status & RENDER_AWAKE_STATUS)); 237 } 238 239 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 240 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 241 242 /* Print media CPG status only if media is present */ 243 if (vcs_mask || vecs_mask) { 244 drm_printf(p, "Media Power Gating Enabled: %s\n", 245 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE)); 246 247 for (n = 0; n < ARRAY_SIZE(media_slices); n++) 248 if (gt->info.engine_mask & media_slices[n].engines) 249 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n, 250 str_up_down(pg_status & media_slices[n].status_bit)); 251 } 252 253 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255) 254 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n", 255 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE)); 256 257 return 0; 258 } 259 260 static ssize_t name_show(struct kobject *kobj, 261 struct kobj_attribute *attr, char *buff) 262 { 263 struct device *dev = kobj_to_dev(kobj); 264 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 265 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 266 ssize_t ret; 267 268 xe_pm_runtime_get(pc_to_xe(pc)); 269 ret = sysfs_emit(buff, "%s\n", gtidle->name); 270 xe_pm_runtime_put(pc_to_xe(pc)); 271 272 return ret; 273 } 274 static struct kobj_attribute name_attr = __ATTR_RO(name); 275 276 static ssize_t idle_status_show(struct kobject *kobj, 277 struct kobj_attribute *attr, char *buff) 278 { 279 struct device *dev = kobj_to_dev(kobj); 280 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 281 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 282 enum xe_gt_idle_state state; 283 284 xe_pm_runtime_get(pc_to_xe(pc)); 285 state = gtidle->idle_status(pc); 286 xe_pm_runtime_put(pc_to_xe(pc)); 287 288 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state)); 289 } 290 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status); 291 292 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle) 293 { 294 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 295 u64 residency; 296 unsigned long flags; 297 298 raw_spin_lock_irqsave(>idle->lock, flags); 299 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc)); 300 raw_spin_unlock_irqrestore(>idle->lock, flags); 301 302 return residency; 303 } 304 305 306 static ssize_t idle_residency_ms_show(struct kobject *kobj, 307 struct kobj_attribute *attr, char *buff) 308 { 309 struct device *dev = kobj_to_dev(kobj); 310 struct xe_gt_idle *gtidle = dev_to_gtidle(dev); 311 struct xe_guc_pc *pc = gtidle_to_pc(gtidle); 312 u64 residency; 313 314 xe_pm_runtime_get(pc_to_xe(pc)); 315 residency = xe_gt_idle_residency_msec(gtidle); 316 xe_pm_runtime_put(pc_to_xe(pc)); 317 318 return sysfs_emit(buff, "%llu\n", residency); 319 } 320 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms); 321 322 static const struct attribute *gt_idle_attrs[] = { 323 &name_attr.attr, 324 &idle_status_attr.attr, 325 &idle_residency_attr.attr, 326 NULL, 327 }; 328 329 static void gt_idle_fini(void *arg) 330 { 331 struct kobject *kobj = arg; 332 struct xe_gt *gt = kobj_to_gt(kobj->parent); 333 334 xe_gt_idle_disable_pg(gt); 335 336 if (gt_to_xe(gt)->info.skip_guc_pc) 337 xe_gt_idle_disable_c6(gt); 338 339 sysfs_remove_files(kobj, gt_idle_attrs); 340 kobject_put(kobj); 341 } 342 343 int xe_gt_idle_init(struct xe_gt_idle *gtidle) 344 { 345 struct xe_gt *gt = gtidle_to_gt(gtidle); 346 struct xe_device *xe = gt_to_xe(gt); 347 struct kobject *kobj; 348 int err; 349 350 if (IS_SRIOV_VF(xe)) 351 return 0; 352 353 kobj = kobject_create_and_add("gtidle", gt->sysfs); 354 if (!kobj) 355 return -ENOMEM; 356 357 raw_spin_lock_init(>idle->lock); 358 359 if (xe_gt_is_media_type(gt)) { 360 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id); 361 gtidle->idle_residency = xe_guc_pc_mc6_residency; 362 } else { 363 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id); 364 gtidle->idle_residency = xe_guc_pc_rc6_residency; 365 } 366 367 /* Multiplier for Residency counter in units of 1.28us */ 368 gtidle->residency_multiplier = 1280; 369 gtidle->idle_status = xe_guc_pc_c_status; 370 371 err = sysfs_create_files(kobj, gt_idle_attrs); 372 if (err) { 373 kobject_put(kobj); 374 return err; 375 } 376 377 xe_gt_idle_enable_pg(gt); 378 379 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj); 380 } 381 382 void xe_gt_idle_enable_c6(struct xe_gt *gt) 383 { 384 xe_device_assert_mem_access(gt_to_xe(gt)); 385 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 386 387 if (IS_SRIOV_VF(gt_to_xe(gt))) 388 return; 389 390 /* Units of 1280 ns for a total of 5s */ 391 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA); 392 /* Enable RC6 */ 393 xe_mmio_write32(>->mmio, RC_CONTROL, 394 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE); 395 } 396 397 int xe_gt_idle_disable_c6(struct xe_gt *gt) 398 { 399 unsigned int fw_ref; 400 401 xe_device_assert_mem_access(gt_to_xe(gt)); 402 403 if (IS_SRIOV_VF(gt_to_xe(gt))) 404 return 0; 405 406 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 407 if (!fw_ref) 408 return -ETIMEDOUT; 409 410 xe_mmio_write32(>->mmio, RC_CONTROL, 0); 411 xe_mmio_write32(>->mmio, RC_STATE, 0); 412 413 xe_force_wake_put(gt_to_fw(gt), fw_ref); 414 415 return 0; 416 } 417