1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_reg_sr.h"
7
8 #include <kunit/visibility.h>
9 #include <linux/align.h>
10 #include <linux/string_helpers.h>
11 #include <linux/xarray.h>
12
13 #include <drm/drm_managed.h>
14 #include <drm/drm_print.h>
15
16 #include "regs/xe_engine_regs.h"
17 #include "regs/xe_gt_regs.h"
18 #include "xe_device_types.h"
19 #include "xe_force_wake.h"
20 #include "xe_gt.h"
21 #include "xe_gt_mcr.h"
22 #include "xe_gt_printk.h"
23 #include "xe_hw_engine_types.h"
24 #include "xe_macros.h"
25 #include "xe_mmio.h"
26 #include "xe_reg_whitelist.h"
27 #include "xe_rtp_types.h"
28
29 #define XE_REG_SR_GROW_STEP_DEFAULT 16
30
reg_sr_fini(struct drm_device * drm,void * arg)31 static void reg_sr_fini(struct drm_device *drm, void *arg)
32 {
33 struct xe_reg_sr *sr = arg;
34
35 xa_destroy(&sr->xa);
36 kfree(sr->pool.arr);
37 memset(&sr->pool, 0, sizeof(sr->pool));
38 }
39
xe_reg_sr_init(struct xe_reg_sr * sr,const char * name,struct xe_device * xe)40 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe)
41 {
42 xa_init(&sr->xa);
43 memset(&sr->pool, 0, sizeof(sr->pool));
44 sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT;
45 sr->name = name;
46
47 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr);
48 }
49 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init);
50
alloc_entry(struct xe_reg_sr * sr)51 static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr)
52 {
53 if (sr->pool.used == sr->pool.allocated) {
54 struct xe_reg_sr_entry *arr;
55
56 arr = krealloc_array(sr->pool.arr,
57 ALIGN(sr->pool.allocated + 1, sr->pool.grow_step),
58 sizeof(*arr), GFP_KERNEL);
59 if (!arr)
60 return NULL;
61
62 sr->pool.arr = arr;
63 sr->pool.allocated += sr->pool.grow_step;
64 }
65
66 return &sr->pool.arr[sr->pool.used++];
67 }
68
compatible_entries(const struct xe_reg_sr_entry * e1,const struct xe_reg_sr_entry * e2)69 static bool compatible_entries(const struct xe_reg_sr_entry *e1,
70 const struct xe_reg_sr_entry *e2)
71 {
72 /*
73 * Don't allow overwriting values: clr_bits/set_bits should be disjoint
74 * when operating in the same register
75 */
76 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits ||
77 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits)
78 return false;
79
80 if (e1->reg.raw != e2->reg.raw)
81 return false;
82
83 return true;
84 }
85
reg_sr_inc_error(struct xe_reg_sr * sr)86 static void reg_sr_inc_error(struct xe_reg_sr *sr)
87 {
88 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
89 sr->errors++;
90 #endif
91 }
92
xe_reg_sr_add(struct xe_reg_sr * sr,const struct xe_reg_sr_entry * e,struct xe_gt * gt)93 int xe_reg_sr_add(struct xe_reg_sr *sr,
94 const struct xe_reg_sr_entry *e,
95 struct xe_gt *gt)
96 {
97 unsigned long idx = e->reg.addr;
98 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);
99 int ret;
100
101 if (pentry) {
102 if (!compatible_entries(pentry, e)) {
103 ret = -EINVAL;
104 goto fail;
105 }
106
107 pentry->clr_bits |= e->clr_bits;
108 pentry->set_bits |= e->set_bits;
109 pentry->read_mask |= e->read_mask;
110
111 return 0;
112 }
113
114 pentry = alloc_entry(sr);
115 if (!pentry) {
116 ret = -ENOMEM;
117 goto fail;
118 }
119
120 *pentry = *e;
121 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));
122 if (ret)
123 goto fail;
124
125 return 0;
126
127 fail:
128 xe_gt_err(gt,
129 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n",
130 idx, e->clr_bits, e->set_bits,
131 str_yes_no(e->reg.masked),
132 str_yes_no(e->reg.mcr),
133 ret);
134 reg_sr_inc_error(sr);
135
136 return ret;
137 }
138
139 /*
140 * Convert back from encoded value to type-safe, only to be used when reg.mcr
141 * is true
142 */
to_xe_reg_mcr(const struct xe_reg reg)143 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
144 {
145 return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
146 }
147
apply_one_mmio(struct xe_gt * gt,struct xe_reg_sr_entry * entry)148 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry)
149 {
150 struct xe_reg reg = entry->reg;
151 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
152 u32 val;
153
154 /*
155 * If this is a masked register, need to set the upper 16 bits.
156 * Set them to clr_bits since that is always a superset of the bits
157 * being modified.
158 *
159 * When it's not masked, we have to read it from hardware, unless we are
160 * supposed to set all bits.
161 */
162 if (reg.masked)
163 val = entry->clr_bits << 16;
164 else if (entry->clr_bits + 1)
165 val = (reg.mcr ?
166 xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
167 xe_mmio_read32(gt, reg)) & (~entry->clr_bits);
168 else
169 val = 0;
170
171 /*
172 * TODO: add selftest to validate all tables, regardless of platform:
173 * - Masked registers can't have set_bits with upper bits set
174 * - set_bits must be contained in clr_bits
175 */
176 val |= entry->set_bits;
177
178 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
179
180 if (entry->reg.mcr)
181 xe_gt_mcr_multicast_write(gt, reg_mcr, val);
182 else
183 xe_mmio_write32(gt, reg, val);
184 }
185
xe_reg_sr_apply_mmio(struct xe_reg_sr * sr,struct xe_gt * gt)186 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
187 {
188 struct xe_reg_sr_entry *entry;
189 unsigned long reg;
190 int err;
191
192 if (xa_empty(&sr->xa))
193 return;
194
195 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
196
197 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL);
198 if (err)
199 goto err_force_wake;
200
201 xa_for_each(&sr->xa, reg, entry)
202 apply_one_mmio(gt, entry);
203
204 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL);
205 XE_WARN_ON(err);
206
207 return;
208
209 err_force_wake:
210 xe_gt_err(gt, "Failed to apply, err=%d\n", err);
211 }
212
xe_reg_sr_apply_whitelist(struct xe_hw_engine * hwe)213 void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe)
214 {
215 struct xe_reg_sr *sr = &hwe->reg_whitelist;
216 struct xe_gt *gt = hwe->gt;
217 struct xe_device *xe = gt_to_xe(gt);
218 struct xe_reg_sr_entry *entry;
219 struct drm_printer p;
220 u32 mmio_base = hwe->mmio_base;
221 unsigned long reg;
222 unsigned int slot = 0;
223 int err;
224
225 if (xa_empty(&sr->xa))
226 return;
227
228 drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name);
229
230 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL);
231 if (err)
232 goto err_force_wake;
233
234 p = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL);
235 xa_for_each(&sr->xa, reg, entry) {
236 if (slot == RING_MAX_NONPRIV_SLOTS) {
237 xe_gt_err(gt,
238 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n",
239 hwe->name, RING_MAX_NONPRIV_SLOTS);
240 break;
241 }
242
243 xe_reg_whitelist_print_entry(&p, 0, reg, entry);
244 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot),
245 reg | entry->set_bits);
246 slot++;
247 }
248
249 /* And clear the rest just in case of garbage */
250 for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) {
251 u32 addr = RING_NOPID(mmio_base).addr;
252
253 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr);
254 }
255
256 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL);
257 XE_WARN_ON(err);
258
259 return;
260
261 err_force_wake:
262 drm_err(&xe->drm, "Failed to apply, err=%d\n", err);
263 }
264
265 /**
266 * xe_reg_sr_dump - print all save/restore entries
267 * @sr: Save/restore entries
268 * @p: DRM printer
269 */
xe_reg_sr_dump(struct xe_reg_sr * sr,struct drm_printer * p)270 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)
271 {
272 struct xe_reg_sr_entry *entry;
273 unsigned long reg;
274
275 if (!sr->name || xa_empty(&sr->xa))
276 return;
277
278 drm_printf(p, "%s\n", sr->name);
279 xa_for_each(&sr->xa, reg, entry)
280 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n",
281 reg, entry->clr_bits, entry->set_bits,
282 str_yes_no(entry->reg.masked),
283 str_yes_no(entry->reg.mcr));
284 }
285