1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_reg_sr.h"
7
8 #include <kunit/visibility.h>
9 #include <linux/align.h>
10 #include <linux/string_helpers.h>
11 #include <linux/xarray.h>
12
13 #include <drm/drm_managed.h>
14 #include <drm/drm_print.h>
15
16 #include "regs/xe_engine_regs.h"
17 #include "regs/xe_gt_regs.h"
18 #include "xe_device.h"
19 #include "xe_device_types.h"
20 #include "xe_force_wake.h"
21 #include "xe_gt.h"
22 #include "xe_gt_mcr.h"
23 #include "xe_gt_printk.h"
24 #include "xe_hw_engine_types.h"
25 #include "xe_macros.h"
26 #include "xe_mmio.h"
27 #include "xe_rtp_types.h"
28
reg_sr_fini(struct drm_device * drm,void * arg)29 static void reg_sr_fini(struct drm_device *drm, void *arg)
30 {
31 struct xe_reg_sr *sr = arg;
32 struct xe_reg_sr_entry *entry;
33 unsigned long reg;
34
35 xa_for_each(&sr->xa, reg, entry)
36 kfree(entry);
37
38 xa_destroy(&sr->xa);
39 }
40
xe_reg_sr_init(struct xe_reg_sr * sr,const char * name,struct xe_device * xe)41 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe)
42 {
43 xa_init(&sr->xa);
44 sr->name = name;
45
46 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr);
47 }
48 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init);
49
compatible_entries(const struct xe_reg_sr_entry * e1,const struct xe_reg_sr_entry * e2)50 static bool compatible_entries(const struct xe_reg_sr_entry *e1,
51 const struct xe_reg_sr_entry *e2)
52 {
53 /*
54 * Don't allow overwriting values: clr_bits/set_bits should be disjoint
55 * when operating in the same register
56 */
57 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits ||
58 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits)
59 return false;
60
61 if (e1->reg.raw != e2->reg.raw)
62 return false;
63
64 return true;
65 }
66
reg_sr_inc_error(struct xe_reg_sr * sr)67 static void reg_sr_inc_error(struct xe_reg_sr *sr)
68 {
69 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
70 sr->errors++;
71 #endif
72 }
73
xe_reg_sr_add(struct xe_reg_sr * sr,const struct xe_reg_sr_entry * e,struct xe_gt * gt)74 int xe_reg_sr_add(struct xe_reg_sr *sr,
75 const struct xe_reg_sr_entry *e,
76 struct xe_gt *gt)
77 {
78 unsigned long idx = e->reg.addr;
79 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);
80 int ret;
81
82 if (pentry) {
83 if (!compatible_entries(pentry, e)) {
84 ret = -EINVAL;
85 goto fail;
86 }
87
88 pentry->clr_bits |= e->clr_bits;
89 pentry->set_bits |= e->set_bits;
90 pentry->read_mask |= e->read_mask;
91
92 return 0;
93 }
94
95 pentry = kmalloc(sizeof(*pentry), GFP_KERNEL);
96 if (!pentry) {
97 ret = -ENOMEM;
98 goto fail;
99 }
100
101 *pentry = *e;
102 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));
103 if (ret)
104 goto fail;
105
106 return 0;
107
108 fail:
109 xe_gt_err(gt,
110 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n",
111 idx, e->clr_bits, e->set_bits,
112 str_yes_no(e->reg.masked),
113 str_yes_no(e->reg.mcr),
114 ret);
115 reg_sr_inc_error(sr);
116
117 return ret;
118 }
119
120 /*
121 * Convert back from encoded value to type-safe, only to be used when reg.mcr
122 * is true
123 */
to_xe_reg_mcr(const struct xe_reg reg)124 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
125 {
126 return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
127 }
128
apply_one_mmio(struct xe_gt * gt,struct xe_reg_sr_entry * entry)129 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry)
130 {
131 struct xe_reg reg = entry->reg;
132 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
133 u32 val;
134
135 /*
136 * If this is a masked register, need to set the upper 16 bits.
137 * Set them to clr_bits since that is always a superset of the bits
138 * being modified.
139 *
140 * When it's not masked, we have to read it from hardware, unless we are
141 * supposed to set all bits.
142 */
143 if (reg.masked)
144 val = entry->clr_bits << 16;
145 else if (entry->clr_bits + 1)
146 val = (reg.mcr ?
147 xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
148 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits);
149 else
150 val = 0;
151
152 /*
153 * TODO: add selftest to validate all tables, regardless of platform:
154 * - Masked registers can't have set_bits with upper bits set
155 * - set_bits must be contained in clr_bits
156 */
157 val |= entry->set_bits;
158
159 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
160
161 if (entry->reg.mcr)
162 xe_gt_mcr_multicast_write(gt, reg_mcr, val);
163 else
164 xe_mmio_write32(>->mmio, reg, val);
165 }
166
xe_reg_sr_apply_mmio(struct xe_reg_sr * sr,struct xe_gt * gt)167 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
168 {
169 struct xe_reg_sr_entry *entry;
170 unsigned long reg;
171 unsigned int fw_ref;
172
173 if (xa_empty(&sr->xa))
174 return;
175
176 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
177
178 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
179 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
180 goto err_force_wake;
181
182 xa_for_each(&sr->xa, reg, entry)
183 apply_one_mmio(gt, entry);
184
185 xe_force_wake_put(gt_to_fw(gt), fw_ref);
186
187 return;
188
189 err_force_wake:
190 xe_force_wake_put(gt_to_fw(gt), fw_ref);
191 xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
192 }
193
194 /**
195 * xe_reg_sr_dump - print all save/restore entries
196 * @sr: Save/restore entries
197 * @p: DRM printer
198 */
xe_reg_sr_dump(struct xe_reg_sr * sr,struct drm_printer * p)199 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)
200 {
201 struct xe_reg_sr_entry *entry;
202 unsigned long reg;
203
204 if (!sr->name || xa_empty(&sr->xa))
205 return;
206
207 drm_printf(p, "%s\n", sr->name);
208 xa_for_each(&sr->xa, reg, entry)
209 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n",
210 reg, entry->clr_bits, entry->set_bits,
211 str_yes_no(entry->reg.masked),
212 str_yes_no(entry->reg.mcr));
213 }
214