1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_rtp.h" 7 8 #include <kunit/visibility.h> 9 10 #include <uapi/drm/xe_drm.h> 11 12 #include "xe_configfs.h" 13 #include "xe_gt.h" 14 #include "xe_gt_topology.h" 15 #include "xe_reg_sr.h" 16 #include "xe_sriov.h" 17 18 /** 19 * DOC: Register Table Processing 20 * 21 * Internal infrastructure to define how registers should be updated based on 22 * rules and actions. This can be used to define tables with multiple entries 23 * (one per register) that will be walked over at some point in time to apply 24 * the values to the registers that have matching rules. 25 */ 26 27 static bool has_samedia(const struct xe_device *xe) 28 { 29 return xe->info.media_verx100 >= 1300; 30 } 31 32 static bool rule_matches(const struct xe_device *xe, 33 struct xe_gt *gt, 34 struct xe_hw_engine *hwe, 35 const struct xe_rtp_rule *rules, 36 unsigned int n_rules) 37 { 38 const struct xe_rtp_rule *r; 39 unsigned int i, rcount = 0; 40 bool match; 41 42 for (r = rules, i = 0; i < n_rules; r = &rules[++i]) { 43 switch (r->match_type) { 44 case XE_RTP_MATCH_OR: 45 /* 46 * This is only reached if a complete set of 47 * rules passed or none were evaluated. For both cases, 48 * shortcut the other rules and return the proper value. 49 */ 50 goto done; 51 case XE_RTP_MATCH_PLATFORM: 52 match = xe->info.platform == r->platform; 53 break; 54 case XE_RTP_MATCH_SUBPLATFORM: 55 match = xe->info.platform == r->platform && 56 xe->info.subplatform == r->subplatform; 57 break; 58 case XE_RTP_MATCH_PLATFORM_STEP: 59 if (drm_WARN_ON(&xe->drm, xe->info.step.platform == STEP_NONE)) 60 return false; 61 62 match = xe->info.step.platform >= r->step_start && 63 xe->info.step.platform < r->step_end; 64 break; 65 case XE_RTP_MATCH_GRAPHICS_VERSION: 66 if (drm_WARN_ON(&xe->drm, !gt)) 67 return false; 68 69 match = xe->info.graphics_verx100 == r->ver_start && 70 (!has_samedia(xe) || !xe_gt_is_media_type(gt)); 71 break; 72 case XE_RTP_MATCH_GRAPHICS_VERSION_RANGE: 73 if (drm_WARN_ON(&xe->drm, !gt)) 74 return false; 75 76 match = xe->info.graphics_verx100 >= r->ver_start && 77 xe->info.graphics_verx100 <= r->ver_end && 78 (!has_samedia(xe) || !xe_gt_is_media_type(gt)); 79 break; 80 case XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT: 81 if (drm_WARN_ON(&xe->drm, !gt)) 82 return false; 83 84 match = xe->info.graphics_verx100 == r->ver_start; 85 break; 86 case XE_RTP_MATCH_GRAPHICS_STEP: 87 if (drm_WARN_ON(&xe->drm, !gt)) 88 return false; 89 90 match = xe->info.step.graphics >= r->step_start && 91 xe->info.step.graphics < r->step_end && 92 (!has_samedia(xe) || !xe_gt_is_media_type(gt)); 93 break; 94 case XE_RTP_MATCH_MEDIA_VERSION: 95 if (drm_WARN_ON(&xe->drm, !gt)) 96 return false; 97 98 match = xe->info.media_verx100 == r->ver_start && 99 (!has_samedia(xe) || xe_gt_is_media_type(gt)); 100 break; 101 case XE_RTP_MATCH_MEDIA_VERSION_RANGE: 102 if (drm_WARN_ON(&xe->drm, !gt)) 103 return false; 104 105 match = xe->info.media_verx100 >= r->ver_start && 106 xe->info.media_verx100 <= r->ver_end && 107 (!has_samedia(xe) || xe_gt_is_media_type(gt)); 108 break; 109 case XE_RTP_MATCH_MEDIA_STEP: 110 if (drm_WARN_ON(&xe->drm, !gt)) 111 return false; 112 113 match = xe->info.step.media >= r->step_start && 114 xe->info.step.media < r->step_end && 115 (!has_samedia(xe) || xe_gt_is_media_type(gt)); 116 break; 117 case XE_RTP_MATCH_MEDIA_VERSION_ANY_GT: 118 if (drm_WARN_ON(&xe->drm, !gt)) 119 return false; 120 121 match = xe->info.media_verx100 == r->ver_start; 122 break; 123 case XE_RTP_MATCH_INTEGRATED: 124 match = !xe->info.is_dgfx; 125 break; 126 case XE_RTP_MATCH_DISCRETE: 127 match = xe->info.is_dgfx; 128 break; 129 case XE_RTP_MATCH_ENGINE_CLASS: 130 if (drm_WARN_ON(&xe->drm, !hwe)) 131 return false; 132 133 match = hwe->class == r->engine_class; 134 break; 135 case XE_RTP_MATCH_NOT_ENGINE_CLASS: 136 if (drm_WARN_ON(&xe->drm, !hwe)) 137 return false; 138 139 match = hwe->class != r->engine_class; 140 break; 141 case XE_RTP_MATCH_FUNC: 142 match = r->match_func(xe, gt, hwe); 143 break; 144 default: 145 drm_warn(&xe->drm, "Invalid RTP match %u\n", 146 r->match_type); 147 match = false; 148 } 149 150 if (!match) { 151 /* 152 * Advance rules until we find XE_RTP_MATCH_OR to check 153 * if there's another set of conditions to check 154 */ 155 while (++i < n_rules && rules[i].match_type != XE_RTP_MATCH_OR) 156 ; 157 158 if (i >= n_rules) 159 return false; 160 161 rcount = 0; 162 } else { 163 rcount++; 164 } 165 } 166 167 done: 168 if (drm_WARN_ON(&xe->drm, !rcount)) 169 return false; 170 171 return true; 172 } 173 174 static void rtp_add_sr_entry(const struct xe_rtp_action *action, 175 struct xe_gt *gt, 176 u32 mmio_base, 177 struct xe_reg_sr *sr) 178 { 179 struct xe_reg_sr_entry sr_entry = { 180 .reg = action->reg, 181 .clr_bits = action->clr_bits, 182 .set_bits = action->set_bits, 183 .read_mask = action->read_mask, 184 }; 185 186 sr_entry.reg.addr += mmio_base; 187 xe_reg_sr_add(sr, &sr_entry, gt); 188 } 189 190 static bool rtp_process_one_sr(const struct xe_rtp_entry_sr *entry, 191 struct xe_device *xe, struct xe_gt *gt, 192 struct xe_hw_engine *hwe, struct xe_reg_sr *sr) 193 { 194 const struct xe_rtp_action *action; 195 u32 mmio_base; 196 unsigned int i; 197 198 if (!rule_matches(xe, gt, hwe, entry->rules, entry->n_rules)) 199 return false; 200 201 for (i = 0, action = &entry->actions[0]; i < entry->n_actions; action++, i++) { 202 if ((entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE) || 203 (action->flags & XE_RTP_ACTION_FLAG_ENGINE_BASE)) 204 mmio_base = hwe->mmio_base; 205 else 206 mmio_base = 0; 207 208 rtp_add_sr_entry(action, gt, mmio_base, sr); 209 } 210 211 return true; 212 } 213 214 static void rtp_get_context(struct xe_rtp_process_ctx *ctx, 215 struct xe_hw_engine **hwe, 216 struct xe_gt **gt, 217 struct xe_device **xe) 218 { 219 switch (ctx->type) { 220 case XE_RTP_PROCESS_TYPE_DEVICE: 221 *hwe = NULL; 222 *gt = NULL; 223 *xe = ctx->xe; 224 break; 225 case XE_RTP_PROCESS_TYPE_GT: 226 *hwe = NULL; 227 *gt = ctx->gt; 228 *xe = gt_to_xe(*gt); 229 break; 230 case XE_RTP_PROCESS_TYPE_ENGINE: 231 *hwe = ctx->hwe; 232 *gt = (*hwe)->gt; 233 *xe = gt_to_xe(*gt); 234 break; 235 } 236 } 237 238 /** 239 * xe_rtp_process_ctx_enable_active_tracking - Enable tracking of active entries 240 * 241 * Set additional metadata to track what entries are considered "active", i.e. 242 * their rules match the condition. Bits are never cleared: entries with 243 * matching rules set the corresponding bit in the bitmap. 244 * 245 * @ctx: The context for processing the table 246 * @active_entries: bitmap to store the active entries 247 * @n_entries: number of entries to be processed 248 */ 249 void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx, 250 unsigned long *active_entries, 251 size_t n_entries) 252 { 253 ctx->active_entries = active_entries; 254 ctx->n_entries = n_entries; 255 } 256 EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_ctx_enable_active_tracking); 257 258 static void rtp_mark_active(struct xe_device *xe, 259 struct xe_rtp_process_ctx *ctx, 260 unsigned int idx) 261 { 262 if (!ctx->active_entries) 263 return; 264 265 if (drm_WARN_ON(&xe->drm, idx >= ctx->n_entries)) 266 return; 267 268 bitmap_set(ctx->active_entries, idx, 1); 269 } 270 271 /** 272 * xe_rtp_process_to_sr - Process all rtp @entries, adding the matching ones to 273 * the save-restore argument. 274 * @ctx: The context for processing the table, with one of device, gt or hwe 275 * @entries: Table with RTP definitions 276 * @n_entries: Number of entries to process, usually ARRAY_SIZE(entries) 277 * @sr: Save-restore struct where matching rules execute the action. This can be 278 * viewed as the "coalesced view" of multiple the tables. The bits for each 279 * register set are expected not to collide with previously added entries 280 * @process_in_vf: Whether this RTP table should get processed for SR-IOV VF 281 * devices. Should generally only be 'true' for LRC tables. 282 * 283 * Walk the table pointed by @entries (with an empty sentinel) and add all 284 * entries with matching rules to @sr. If @hwe is not NULL, its mmio_base is 285 * used to calculate the right register offset 286 */ 287 void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, 288 const struct xe_rtp_entry_sr *entries, 289 size_t n_entries, 290 struct xe_reg_sr *sr, 291 bool process_in_vf) 292 { 293 const struct xe_rtp_entry_sr *entry; 294 struct xe_hw_engine *hwe = NULL; 295 struct xe_gt *gt = NULL; 296 struct xe_device *xe = NULL; 297 298 rtp_get_context(ctx, &hwe, >, &xe); 299 300 if (!process_in_vf && IS_SRIOV_VF(xe)) 301 return; 302 303 xe_assert(xe, entries); 304 305 for (entry = entries; entry - entries < n_entries; entry++) { 306 bool match = false; 307 308 if (entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE) { 309 struct xe_hw_engine *each_hwe; 310 enum xe_hw_engine_id id; 311 312 for_each_hw_engine(each_hwe, gt, id) 313 match |= rtp_process_one_sr(entry, xe, gt, 314 each_hwe, sr); 315 } else { 316 match = rtp_process_one_sr(entry, xe, gt, hwe, sr); 317 } 318 319 if (match) 320 rtp_mark_active(xe, ctx, entry - entries); 321 } 322 } 323 EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_to_sr); 324 325 /** 326 * xe_rtp_process - Process all rtp @entries, without running any action 327 * @ctx: The context for processing the table, with one of device, gt or hwe 328 * @entries: Table with RTP definitions 329 * 330 * Walk the table pointed by @entries (with an empty sentinel), executing the 331 * rules. One difference from xe_rtp_process_to_sr(): there is no action 332 * associated with each entry since this uses struct xe_rtp_entry. Its main use 333 * is for marking active workarounds via 334 * xe_rtp_process_ctx_enable_active_tracking(). 335 */ 336 void xe_rtp_process(struct xe_rtp_process_ctx *ctx, 337 const struct xe_rtp_entry *entries) 338 { 339 const struct xe_rtp_entry *entry; 340 struct xe_hw_engine *hwe; 341 struct xe_gt *gt; 342 struct xe_device *xe; 343 344 rtp_get_context(ctx, &hwe, >, &xe); 345 346 for (entry = entries; entry && entry->rules; entry++) { 347 if (!rule_matches(xe, gt, hwe, entry->rules, entry->n_rules)) 348 continue; 349 350 rtp_mark_active(xe, ctx, entry - entries); 351 } 352 } 353 EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process); 354 355 bool xe_rtp_match_even_instance(const struct xe_device *xe, 356 const struct xe_gt *gt, 357 const struct xe_hw_engine *hwe) 358 { 359 return hwe->instance % 2 == 0; 360 } 361 362 bool xe_rtp_match_first_render_or_compute(const struct xe_device *xe, 363 const struct xe_gt *gt, 364 const struct xe_hw_engine *hwe) 365 { 366 u64 render_compute_mask = gt->info.engine_mask & 367 (XE_HW_ENGINE_CCS_MASK | XE_HW_ENGINE_RCS_MASK); 368 369 return render_compute_mask && 370 hwe->engine_id == __ffs(render_compute_mask); 371 } 372 373 bool xe_rtp_match_not_sriov_vf(const struct xe_device *xe, 374 const struct xe_gt *gt, 375 const struct xe_hw_engine *hwe) 376 { 377 return !IS_SRIOV_VF(xe); 378 } 379 380 bool xe_rtp_match_psmi_enabled(const struct xe_device *xe, 381 const struct xe_gt *gt, 382 const struct xe_hw_engine *hwe) 383 { 384 return xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev)); 385 } 386 387 bool xe_rtp_match_gt_has_discontiguous_dss_groups(const struct xe_device *xe, 388 const struct xe_gt *gt, 389 const struct xe_hw_engine *hwe) 390 { 391 return xe_gt_has_discontiguous_dss_groups(gt); 392 } 393 394 bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, 395 const struct xe_gt *gt, 396 const struct xe_hw_engine *hwe) 397 { 398 return xe->info.has_flat_ccs; 399 } 400