1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "xe_reg_whitelist.h" 7 8 #include <kunit/visibility.h> 9 10 #include "regs/xe_engine_regs.h" 11 #include "regs/xe_gt_regs.h" 12 #include "regs/xe_oa_regs.h" 13 #include "xe_device.h" 14 #include "xe_gt.h" 15 #include "xe_gt_types.h" 16 #include "xe_gt_printk.h" 17 #include "xe_platform_types.h" 18 #include "xe_reg_sr.h" 19 #include "xe_rtp.h" 20 #include "xe_step.h" 21 22 #undef XE_REG_MCR 23 #define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) 24 25 static bool match_not_render(const struct xe_device *xe, 26 const struct xe_gt *gt, 27 const struct xe_hw_engine *hwe) 28 { 29 return hwe->class != XE_ENGINE_CLASS_RENDER; 30 } 31 32 static bool match_has_mert(const struct xe_device *xe, 33 const struct xe_gt *gt, 34 const struct xe_hw_engine *hwe) 35 { 36 return xe_device_has_mert((struct xe_device *)xe); 37 } 38 39 static bool match_multi_queue_class(const struct xe_device *xe, 40 const struct xe_gt *gt, 41 const struct xe_hw_engine *hwe) 42 { 43 return xe_gt_supports_multi_queue(gt, hwe->class); 44 } 45 46 VISIBLE_IF_KUNIT const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR( 47 { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), 48 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), 49 XE_RTP_ACTIONS(WHITELIST(PS_INVOCATION_COUNT, 50 RING_FORCE_TO_NONPRIV_ACCESS_RD | 51 RING_FORCE_TO_NONPRIV_RANGE_4)) 52 }, 53 { XE_RTP_NAME("1508744258, 14012131227, 1808121037"), 54 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), 55 XE_RTP_ACTIONS(WHITELIST(COMMON_SLICE_CHICKEN1, 0)) 56 }, 57 { XE_RTP_NAME("1806527549"), 58 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), 59 XE_RTP_ACTIONS(WHITELIST(HIZ_CHICKEN, 0)) 60 }, 61 { XE_RTP_NAME("allow_read_ctx_timestamp"), 62 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260), FUNC(match_not_render)), 63 XE_RTP_ACTIONS(WHITELIST(RING_CTX_TIMESTAMP(0), 64 RING_FORCE_TO_NONPRIV_ACCESS_RD, 65 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 66 }, 67 { XE_RTP_NAME("allow_read_queue_timestamp"), 68 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)), 69 XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0), 70 RING_FORCE_TO_NONPRIV_ACCESS_RD, 71 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 72 }, 73 { XE_RTP_NAME("16014440446"), 74 XE_RTP_RULES(PLATFORM(PVC)), 75 XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400), 76 RING_FORCE_TO_NONPRIV_DENY | 77 RING_FORCE_TO_NONPRIV_RANGE_64), 78 WHITELIST(XE_REG(0x4500), 79 RING_FORCE_TO_NONPRIV_DENY | 80 RING_FORCE_TO_NONPRIV_RANGE_64)) 81 }, 82 { XE_RTP_NAME("16017236439"), 83 XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COPY)), 84 XE_RTP_ACTIONS(WHITELIST(BCS_SWCTRL(0), 85 RING_FORCE_TO_NONPRIV_DENY, 86 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 87 }, 88 { XE_RTP_NAME("16020183090"), 89 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), 90 ENGINE_CLASS(RENDER)), 91 XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0)) 92 }, 93 { XE_RTP_NAME("14024997852"), 94 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), ENGINE_CLASS(RENDER)), 95 XE_RTP_ACTIONS(WHITELIST(FF_MODE, 96 RING_FORCE_TO_NONPRIV_ACCESS_RW), 97 WHITELIST(VFLSKPD, 98 RING_FORCE_TO_NONPRIV_ACCESS_RW)) 99 }, 100 { XE_RTP_NAME("14024997852"), 101 XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), 102 ENGINE_CLASS(RENDER)), 103 XE_RTP_ACTIONS(WHITELIST(FF_MODE, 104 RING_FORCE_TO_NONPRIV_ACCESS_RW), 105 WHITELIST(VFLSKPD, 106 RING_FORCE_TO_NONPRIV_ACCESS_RW)) 107 }, 108 ); 109 EXPORT_SYMBOL_IF_KUNIT(register_whitelist); 110 111 static const struct xe_rtp_table_sr oa_whitelist = XE_RTP_TABLE_SR( 112 113 #define WHITELIST_DENY(r, f) WHITELIST(r, (f) | RING_FORCE_TO_NONPRIV_DENY) 114 115 #define WHITELIST_OA_MMIO_TRG(trg, status, head) \ 116 WHITELIST_DENY(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \ 117 WHITELIST_DENY(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \ 118 WHITELIST_DENY(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4) 119 120 #define WHITELIST_OAG_MMIO_TRG \ 121 WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR) 122 123 #define WHITELIST_OAM_MMIO_TRG \ 124 WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \ 125 OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \ 126 OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \ 127 WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \ 128 OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \ 129 OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \ 130 WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \ 131 OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \ 132 OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ)) 133 134 #define WHITELIST_OA_MERT_MMIO_TRG \ 135 WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_TAIL_POINTER) 136 137 { XE_RTP_NAME("oag_mmio_trg_rcs"), 138 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), 139 ENGINE_CLASS(RENDER)), 140 XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) 141 }, 142 { XE_RTP_NAME("oag_mmio_trg_ccs"), 143 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), 144 ENGINE_CLASS(COMPUTE)), 145 XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) 146 }, 147 { XE_RTP_NAME("oam_mmio_trg_vcs"), 148 XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED), 149 ENGINE_CLASS(VIDEO_DECODE)), 150 XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) 151 }, 152 { XE_RTP_NAME("oam_mmio_trg_vecs"), 153 XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED), 154 ENGINE_CLASS(VIDEO_ENHANCE)), 155 XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) 156 }, 157 { XE_RTP_NAME("oa_mert_mmio_trg_ccs"), 158 XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)), 159 XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) 160 }, 161 { XE_RTP_NAME("oa_mert_mmio_trg_bcs"), 162 XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)), 163 XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) 164 }, 165 ); 166 167 static int whitelist_apply_to_hwe(struct xe_hw_engine *hwe, struct xe_reg_sr *in, 168 struct xe_reg_sr *out, int first_slot) 169 { 170 struct xe_reg_sr *sr = in; 171 struct xe_reg_sr_entry *entry; 172 struct drm_printer p; 173 unsigned long reg; 174 unsigned int slot; 175 176 xe_gt_dbg(hwe->gt, "Add %s whitelist to engine\n", sr->name); 177 p = xe_gt_dbg_printer(hwe->gt); 178 179 slot = first_slot; 180 xa_for_each(&sr->xa, reg, entry) { 181 struct xe_reg_sr_entry hwe_entry = { 182 .reg = RING_FORCE_TO_NONPRIV(hwe->mmio_base, slot), 183 .set_bits = entry->reg.addr | entry->set_bits, 184 .clr_bits = ~0u, 185 .read_mask = entry->read_mask, 186 }; 187 188 if (slot == RING_MAX_NONPRIV_SLOTS) { 189 xe_gt_err(hwe->gt, 190 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n", 191 hwe->name, RING_MAX_NONPRIV_SLOTS); 192 break; 193 } 194 195 xe_reg_whitelist_print_entry(&p, 0, reg, entry); 196 xe_reg_sr_add(out, &hwe_entry, hwe->gt); 197 198 slot++; 199 } 200 201 return slot; 202 } 203 204 /** 205 * xe_reg_whitelist_process_engine - process table of registers to whitelist 206 * @hwe: engine instance to process whitelist for 207 * 208 * Process wwhitelist table for this platform, saving in @hwe all the 209 * registers that need to be whitelisted by the hardware so they can be accessed 210 * by userspace. 211 */ 212 void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe) 213 { 214 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); 215 int first_oa_slot; 216 217 xe_rtp_process_to_sr(&ctx, ®ister_whitelist, &hwe->reg_whitelist, false); 218 first_oa_slot = whitelist_apply_to_hwe(hwe, &hwe->reg_whitelist, &hwe->reg_sr, 0); 219 220 xe_rtp_process_to_sr(&ctx, &oa_whitelist, &hwe->oa_whitelist, false); 221 222 /* 223 * Save oa nonpriv registers to hwe->oa_sr, from which oa registers are whitelisted 224 * or de-whitelisted, by toggling the 'deny' bit on oa stream open/close 225 */ 226 whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->oa_sr, first_oa_slot); 227 228 /* 229 * Also save oa nonpriv registers to hwe->reg_sr, to ensure oa registers are not 230 * whitelisted by default after probe, gt reset, resume and engine reset 231 */ 232 whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->reg_sr, first_oa_slot); 233 } 234 235 static void __whitelist_oa_regs(struct xe_hw_engine *hwe, bool whitelist) 236 { 237 struct xe_reg_sr_entry *entry; 238 unsigned long reg; 239 240 xa_for_each(&hwe->oa_sr.xa, reg, entry) { 241 if (whitelist) 242 entry->set_bits &= ~RING_FORCE_TO_NONPRIV_DENY; 243 else 244 entry->set_bits |= RING_FORCE_TO_NONPRIV_DENY; 245 } 246 247 xe_reg_sr_apply_mmio(&hwe->oa_sr, hwe->gt); 248 } 249 250 /** 251 * xe_reg_whitelist_oa_regs - whitelist oa registers for gt 252 * @gt: gt to whitelist oa registers for 253 * 254 * Whitelist OA registers by resetting RING_FORCE_TO_NONPRIV_DENY 255 */ 256 void xe_reg_whitelist_oa_regs(struct xe_gt *gt) 257 { 258 struct xe_hw_engine *hwe; 259 enum xe_hw_engine_id id; 260 261 lockdep_assert_held(>->oa.gt_lock); 262 if (gt->oa.whitelist_count++) 263 return; 264 265 for_each_hw_engine(hwe, gt, id) 266 __whitelist_oa_regs(hwe, true); 267 } 268 269 /** 270 * xe_reg_dewhitelist_oa_regs - dewhitelist oa registers for gt 271 * @gt: gt to dewhitelist oa registers for 272 * 273 * Dewhitelist OA registers by setting RING_FORCE_TO_NONPRIV_DENY 274 */ 275 void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt) 276 { 277 struct xe_hw_engine *hwe; 278 enum xe_hw_engine_id id; 279 280 lockdep_assert_held(>->oa.gt_lock); 281 xe_assert(gt_to_xe(gt), gt->oa.whitelist_count); 282 if (--gt->oa.whitelist_count) 283 return; 284 285 for_each_hw_engine(hwe, gt, id) 286 __whitelist_oa_regs(hwe, false); 287 } 288 289 /** 290 * xe_reg_whitelist_print_entry - print one whitelist entry 291 * @p: DRM printer 292 * @indent: indent level 293 * @reg: register allowed/denied 294 * @entry: save-restore entry 295 * 296 * Print details about the entry added to allow/deny access 297 */ 298 void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, 299 u32 reg, struct xe_reg_sr_entry *entry) 300 { 301 u32 val = entry->set_bits; 302 const char *access_str = "(invalid)"; 303 unsigned int range_bit = 2; 304 u32 range_start, range_end; 305 bool deny; 306 307 deny = val & RING_FORCE_TO_NONPRIV_DENY; 308 309 switch (val & RING_FORCE_TO_NONPRIV_RANGE_MASK) { 310 case RING_FORCE_TO_NONPRIV_RANGE_4: 311 range_bit = 4; 312 break; 313 case RING_FORCE_TO_NONPRIV_RANGE_16: 314 range_bit = 6; 315 break; 316 case RING_FORCE_TO_NONPRIV_RANGE_64: 317 range_bit = 8; 318 break; 319 } 320 321 range_start = reg & REG_GENMASK(25, range_bit); 322 range_end = range_start | REG_GENMASK(range_bit - 1, 0); 323 324 switch (val & RING_FORCE_TO_NONPRIV_ACCESS_MASK) { 325 case RING_FORCE_TO_NONPRIV_ACCESS_RW: 326 access_str = "rw"; 327 break; 328 case RING_FORCE_TO_NONPRIV_ACCESS_RD: 329 access_str = "read"; 330 break; 331 case RING_FORCE_TO_NONPRIV_ACCESS_WR: 332 access_str = "write"; 333 break; 334 } 335 336 drm_printf_indent(p, indent, "REG[0x%x-0x%x]: %s %s access\n", 337 range_start, range_end, 338 deny ? "deny" : "allow", 339 access_str); 340 } 341 342 /** 343 * xe_reg_whitelist_dump - print all whitelist entries 344 * @sr: Save/restore entries 345 * @p: DRM printer 346 */ 347 void xe_reg_whitelist_dump(struct xe_reg_sr *sr, struct drm_printer *p) 348 { 349 struct xe_reg_sr_entry *entry; 350 unsigned long reg; 351 352 if (!sr->name || xa_empty(&sr->xa)) 353 return; 354 355 drm_printf(p, "%s\n", sr->name); 356 xa_for_each(&sr->xa, reg, entry) 357 xe_reg_whitelist_print_entry(p, 1, reg, entry); 358 } 359