1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include "xe_reg_whitelist.h"
7
8 #include <kunit/visibility.h>
9
10 #include "regs/xe_engine_regs.h"
11 #include "regs/xe_gt_regs.h"
12 #include "regs/xe_oa_regs.h"
13 #include "xe_device.h"
14 #include "xe_gt.h"
15 #include "xe_gt_types.h"
16 #include "xe_gt_printk.h"
17 #include "xe_platform_types.h"
18 #include "xe_reg_sr.h"
19 #include "xe_rtp.h"
20 #include "xe_step.h"
21
22 #undef XE_REG_MCR
23 #define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1)
24
match_not_render(const struct xe_device * xe,const struct xe_gt * gt,const struct xe_hw_engine * hwe)25 static bool match_not_render(const struct xe_device *xe,
26 const struct xe_gt *gt,
27 const struct xe_hw_engine *hwe)
28 {
29 return hwe->class != XE_ENGINE_CLASS_RENDER;
30 }
31
match_has_mert(const struct xe_device * xe,const struct xe_gt * gt,const struct xe_hw_engine * hwe)32 static bool match_has_mert(const struct xe_device *xe,
33 const struct xe_gt *gt,
34 const struct xe_hw_engine *hwe)
35 {
36 return xe_device_has_mert((struct xe_device *)xe);
37 }
38
match_multi_queue_class(const struct xe_device * xe,const struct xe_gt * gt,const struct xe_hw_engine * hwe)39 static bool match_multi_queue_class(const struct xe_device *xe,
40 const struct xe_gt *gt,
41 const struct xe_hw_engine *hwe)
42 {
43 return xe_gt_supports_multi_queue(gt, hwe->class);
44 }
45
46 VISIBLE_IF_KUNIT const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR(
47 { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),
48 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
49 XE_RTP_ACTIONS(WHITELIST(PS_INVOCATION_COUNT,
50 RING_FORCE_TO_NONPRIV_ACCESS_RD |
51 RING_FORCE_TO_NONPRIV_RANGE_4))
52 },
53 { XE_RTP_NAME("1508744258, 14012131227, 1808121037"),
54 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
55 XE_RTP_ACTIONS(WHITELIST(COMMON_SLICE_CHICKEN1, 0))
56 },
57 { XE_RTP_NAME("1806527549"),
58 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
59 XE_RTP_ACTIONS(WHITELIST(HIZ_CHICKEN, 0))
60 },
61 { XE_RTP_NAME("allow_read_ctx_timestamp"),
62 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260), FUNC(match_not_render)),
63 XE_RTP_ACTIONS(WHITELIST(RING_CTX_TIMESTAMP(0),
64 RING_FORCE_TO_NONPRIV_ACCESS_RD,
65 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
66 },
67 { XE_RTP_NAME("allow_read_queue_timestamp"),
68 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)),
69 XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0),
70 RING_FORCE_TO_NONPRIV_ACCESS_RD,
71 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
72 },
73 { XE_RTP_NAME("16014440446"),
74 XE_RTP_RULES(PLATFORM(PVC)),
75 XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400),
76 RING_FORCE_TO_NONPRIV_DENY |
77 RING_FORCE_TO_NONPRIV_RANGE_64),
78 WHITELIST(XE_REG(0x4500),
79 RING_FORCE_TO_NONPRIV_DENY |
80 RING_FORCE_TO_NONPRIV_RANGE_64))
81 },
82 { XE_RTP_NAME("16017236439"),
83 XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COPY)),
84 XE_RTP_ACTIONS(WHITELIST(BCS_SWCTRL(0),
85 RING_FORCE_TO_NONPRIV_DENY,
86 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
87 },
88 { XE_RTP_NAME("16020183090"),
89 XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
90 ENGINE_CLASS(RENDER)),
91 XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0))
92 },
93 { XE_RTP_NAME("14024997852"),
94 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3005), ENGINE_CLASS(RENDER)),
95 XE_RTP_ACTIONS(WHITELIST(FF_MODE,
96 RING_FORCE_TO_NONPRIV_ACCESS_RW),
97 WHITELIST(VFLSKPD,
98 RING_FORCE_TO_NONPRIV_ACCESS_RW))
99 },
100 { XE_RTP_NAME("14024997852"),
101 XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
102 ENGINE_CLASS(RENDER)),
103 XE_RTP_ACTIONS(WHITELIST(FF_MODE,
104 RING_FORCE_TO_NONPRIV_ACCESS_RW),
105 WHITELIST(VFLSKPD,
106 RING_FORCE_TO_NONPRIV_ACCESS_RW))
107 },
108 );
109 EXPORT_SYMBOL_IF_KUNIT(register_whitelist);
110
111 static const struct xe_rtp_table_sr oa_whitelist = XE_RTP_TABLE_SR(
112
113 #define WHITELIST_DENY(r, f) WHITELIST(r, (f) | RING_FORCE_TO_NONPRIV_DENY)
114
115 #define WHITELIST_OA_MMIO_TRG(trg, status, head) \
116 WHITELIST_DENY(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \
117 WHITELIST_DENY(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \
118 WHITELIST_DENY(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4)
119
120 #define WHITELIST_OAG_MMIO_TRG \
121 WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR)
122
123 #define WHITELIST_OAM_MMIO_TRG \
124 WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \
125 OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \
126 OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \
127 WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \
128 OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \
129 OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \
130 WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \
131 OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \
132 OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ))
133
134 #define WHITELIST_OA_MERT_MMIO_TRG \
135 WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_TAIL_POINTER)
136
137 { XE_RTP_NAME("oag_mmio_trg_rcs"),
138 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
139 ENGINE_CLASS(RENDER)),
140 XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
141 },
142 { XE_RTP_NAME("oag_mmio_trg_ccs"),
143 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
144 ENGINE_CLASS(COMPUTE)),
145 XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
146 },
147 { XE_RTP_NAME("oam_mmio_trg_vcs"),
148 XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
149 ENGINE_CLASS(VIDEO_DECODE)),
150 XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
151 },
152 { XE_RTP_NAME("oam_mmio_trg_vecs"),
153 XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
154 ENGINE_CLASS(VIDEO_ENHANCE)),
155 XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
156 },
157 { XE_RTP_NAME("oa_mert_mmio_trg_ccs"),
158 XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)),
159 XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
160 },
161 { XE_RTP_NAME("oa_mert_mmio_trg_bcs"),
162 XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)),
163 XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
164 },
165 );
166
whitelist_apply_to_hwe(struct xe_hw_engine * hwe,struct xe_reg_sr * in,struct xe_reg_sr * out,int first_slot)167 static int whitelist_apply_to_hwe(struct xe_hw_engine *hwe, struct xe_reg_sr *in,
168 struct xe_reg_sr *out, int first_slot)
169 {
170 struct xe_reg_sr *sr = in;
171 struct xe_reg_sr_entry *entry;
172 struct drm_printer p;
173 unsigned long reg;
174 unsigned int slot;
175
176 xe_gt_dbg(hwe->gt, "Add %s whitelist to engine\n", sr->name);
177 p = xe_gt_dbg_printer(hwe->gt);
178
179 slot = first_slot;
180 xa_for_each(&sr->xa, reg, entry) {
181 struct xe_reg_sr_entry hwe_entry = {
182 .reg = RING_FORCE_TO_NONPRIV(hwe->mmio_base, slot),
183 .set_bits = entry->reg.addr | entry->set_bits,
184 .clr_bits = ~0u,
185 .read_mask = entry->read_mask,
186 };
187
188 if (slot == RING_MAX_NONPRIV_SLOTS) {
189 xe_gt_err(hwe->gt,
190 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n",
191 hwe->name, RING_MAX_NONPRIV_SLOTS);
192 break;
193 }
194
195 xe_reg_whitelist_print_entry(&p, 0, reg, entry);
196 xe_reg_sr_add(out, &hwe_entry, hwe->gt);
197
198 slot++;
199 }
200
201 return slot;
202 }
203
204 /**
205 * xe_reg_whitelist_process_engine - process table of registers to whitelist
206 * @hwe: engine instance to process whitelist for
207 *
208 * Process wwhitelist table for this platform, saving in @hwe all the
209 * registers that need to be whitelisted by the hardware so they can be accessed
210 * by userspace.
211 */
xe_reg_whitelist_process_engine(struct xe_hw_engine * hwe)212 void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe)
213 {
214 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
215 int first_oa_slot;
216
217 xe_rtp_process_to_sr(&ctx, ®ister_whitelist, &hwe->reg_whitelist, false);
218 first_oa_slot = whitelist_apply_to_hwe(hwe, &hwe->reg_whitelist, &hwe->reg_sr, 0);
219
220 xe_rtp_process_to_sr(&ctx, &oa_whitelist, &hwe->oa_whitelist, false);
221
222 /*
223 * Save oa nonpriv registers to hwe->oa_sr, from which oa registers are whitelisted
224 * or de-whitelisted, by toggling the 'deny' bit on oa stream open/close
225 */
226 whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->oa_sr, first_oa_slot);
227
228 /*
229 * Also save oa nonpriv registers to hwe->reg_sr, to ensure oa registers are not
230 * whitelisted by default after probe, gt reset, resume and engine reset
231 */
232 whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->reg_sr, first_oa_slot);
233 }
234
__whitelist_oa_regs(struct xe_hw_engine * hwe,bool whitelist)235 static void __whitelist_oa_regs(struct xe_hw_engine *hwe, bool whitelist)
236 {
237 struct xe_reg_sr_entry *entry;
238 unsigned long reg;
239
240 xa_for_each(&hwe->oa_sr.xa, reg, entry) {
241 if (whitelist)
242 entry->set_bits &= ~RING_FORCE_TO_NONPRIV_DENY;
243 else
244 entry->set_bits |= RING_FORCE_TO_NONPRIV_DENY;
245 }
246
247 xe_reg_sr_apply_mmio(&hwe->oa_sr, hwe->gt);
248 }
249
250 /**
251 * xe_reg_whitelist_oa_regs - whitelist oa registers for gt
252 * @gt: gt to whitelist oa registers for
253 *
254 * Whitelist OA registers by resetting RING_FORCE_TO_NONPRIV_DENY
255 */
xe_reg_whitelist_oa_regs(struct xe_gt * gt)256 void xe_reg_whitelist_oa_regs(struct xe_gt *gt)
257 {
258 struct xe_hw_engine *hwe;
259 enum xe_hw_engine_id id;
260
261 lockdep_assert_held(>->oa.gt_lock);
262 if (gt->oa.whitelist_count++)
263 return;
264
265 for_each_hw_engine(hwe, gt, id)
266 __whitelist_oa_regs(hwe, true);
267 }
268
269 /**
270 * xe_reg_dewhitelist_oa_regs - dewhitelist oa registers for gt
271 * @gt: gt to dewhitelist oa registers for
272 *
273 * Dewhitelist OA registers by setting RING_FORCE_TO_NONPRIV_DENY
274 */
xe_reg_dewhitelist_oa_regs(struct xe_gt * gt)275 void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt)
276 {
277 struct xe_hw_engine *hwe;
278 enum xe_hw_engine_id id;
279
280 lockdep_assert_held(>->oa.gt_lock);
281 xe_assert(gt_to_xe(gt), gt->oa.whitelist_count);
282 if (--gt->oa.whitelist_count)
283 return;
284
285 for_each_hw_engine(hwe, gt, id)
286 __whitelist_oa_regs(hwe, false);
287 }
288
289 /**
290 * xe_reg_whitelist_print_entry - print one whitelist entry
291 * @p: DRM printer
292 * @indent: indent level
293 * @reg: register allowed/denied
294 * @entry: save-restore entry
295 *
296 * Print details about the entry added to allow/deny access
297 */
xe_reg_whitelist_print_entry(struct drm_printer * p,unsigned int indent,u32 reg,struct xe_reg_sr_entry * entry)298 void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent,
299 u32 reg, struct xe_reg_sr_entry *entry)
300 {
301 u32 val = entry->set_bits;
302 const char *access_str = "(invalid)";
303 unsigned int range_bit = 2;
304 u32 range_start, range_end;
305 bool deny;
306
307 deny = val & RING_FORCE_TO_NONPRIV_DENY;
308
309 switch (val & RING_FORCE_TO_NONPRIV_RANGE_MASK) {
310 case RING_FORCE_TO_NONPRIV_RANGE_4:
311 range_bit = 4;
312 break;
313 case RING_FORCE_TO_NONPRIV_RANGE_16:
314 range_bit = 6;
315 break;
316 case RING_FORCE_TO_NONPRIV_RANGE_64:
317 range_bit = 8;
318 break;
319 }
320
321 range_start = reg & REG_GENMASK(25, range_bit);
322 range_end = range_start | REG_GENMASK(range_bit - 1, 0);
323
324 switch (val & RING_FORCE_TO_NONPRIV_ACCESS_MASK) {
325 case RING_FORCE_TO_NONPRIV_ACCESS_RW:
326 access_str = "rw";
327 break;
328 case RING_FORCE_TO_NONPRIV_ACCESS_RD:
329 access_str = "read";
330 break;
331 case RING_FORCE_TO_NONPRIV_ACCESS_WR:
332 access_str = "write";
333 break;
334 }
335
336 drm_printf_indent(p, indent, "REG[0x%x-0x%x]: %s %s access\n",
337 range_start, range_end,
338 deny ? "deny" : "allow",
339 access_str);
340 }
341
342 /**
343 * xe_reg_whitelist_dump - print all whitelist entries
344 * @sr: Save/restore entries
345 * @p: DRM printer
346 */
xe_reg_whitelist_dump(struct xe_reg_sr * sr,struct drm_printer * p)347 void xe_reg_whitelist_dump(struct xe_reg_sr *sr, struct drm_printer *p)
348 {
349 struct xe_reg_sr_entry *entry;
350 unsigned long reg;
351
352 if (!sr->name || xa_empty(&sr->xa))
353 return;
354
355 drm_printf(p, "%s\n", sr->name);
356 xa_for_each(&sr->xa, reg, entry)
357 xe_reg_whitelist_print_entry(p, 1, reg, entry);
358 }
359