1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <kunit/visibility.h> 9 #include <linux/align.h> 10 #include <linux/string_helpers.h> 11 #include <linux/xarray.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "regs/xe_engine_regs.h" 17 #include "regs/xe_gt_regs.h" 18 #include "xe_device_types.h" 19 #include "xe_force_wake.h" 20 #include "xe_gt.h" 21 #include "xe_gt_mcr.h" 22 #include "xe_gt_printk.h" 23 #include "xe_hw_engine_types.h" 24 #include "xe_macros.h" 25 #include "xe_mmio.h" 26 #include "xe_reg_whitelist.h" 27 #include "xe_rtp_types.h" 28 29 #define XE_REG_SR_GROW_STEP_DEFAULT 16 30 31 static void reg_sr_fini(struct drm_device *drm, void *arg) 32 { 33 struct xe_reg_sr *sr = arg; 34 35 xa_destroy(&sr->xa); 36 kfree(sr->pool.arr); 37 memset(&sr->pool, 0, sizeof(sr->pool)); 38 } 39 40 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 41 { 42 xa_init(&sr->xa); 43 memset(&sr->pool, 0, sizeof(sr->pool)); 44 sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT; 45 sr->name = name; 46 47 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 48 } 49 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); 50 51 static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr) 52 { 53 if (sr->pool.used == sr->pool.allocated) { 54 struct xe_reg_sr_entry *arr; 55 56 arr = krealloc_array(sr->pool.arr, 57 ALIGN(sr->pool.allocated + 1, sr->pool.grow_step), 58 sizeof(*arr), GFP_KERNEL); 59 if (!arr) 60 return NULL; 61 62 sr->pool.arr = arr; 63 sr->pool.allocated += sr->pool.grow_step; 64 } 65 66 return &sr->pool.arr[sr->pool.used++]; 67 } 68 69 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 70 const struct xe_reg_sr_entry *e2) 71 { 72 /* 73 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 74 * when operating in the same register 75 */ 76 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 77 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 78 return false; 79 80 if (e1->reg.raw != e2->reg.raw) 81 return false; 82 83 return true; 84 } 85 86 static void reg_sr_inc_error(struct xe_reg_sr *sr) 87 { 88 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 89 sr->errors++; 90 #endif 91 } 92 93 int xe_reg_sr_add(struct xe_reg_sr *sr, 94 const struct xe_reg_sr_entry *e, 95 struct xe_gt *gt) 96 { 97 unsigned long idx = e->reg.addr; 98 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 99 int ret; 100 101 if (pentry) { 102 if (!compatible_entries(pentry, e)) { 103 ret = -EINVAL; 104 goto fail; 105 } 106 107 pentry->clr_bits |= e->clr_bits; 108 pentry->set_bits |= e->set_bits; 109 pentry->read_mask |= e->read_mask; 110 111 return 0; 112 } 113 114 pentry = alloc_entry(sr); 115 if (!pentry) { 116 ret = -ENOMEM; 117 goto fail; 118 } 119 120 *pentry = *e; 121 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 122 if (ret) 123 goto fail; 124 125 return 0; 126 127 fail: 128 xe_gt_err(gt, 129 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", 130 idx, e->clr_bits, e->set_bits, 131 str_yes_no(e->reg.masked), 132 str_yes_no(e->reg.mcr), 133 ret); 134 reg_sr_inc_error(sr); 135 136 return ret; 137 } 138 139 /* 140 * Convert back from encoded value to type-safe, only to be used when reg.mcr 141 * is true 142 */ 143 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) 144 { 145 return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; 146 } 147 148 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) 149 { 150 struct xe_reg reg = entry->reg; 151 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); 152 u32 val; 153 154 /* 155 * If this is a masked register, need to set the upper 16 bits. 156 * Set them to clr_bits since that is always a superset of the bits 157 * being modified. 158 * 159 * When it's not masked, we have to read it from hardware, unless we are 160 * supposed to set all bits. 161 */ 162 if (reg.masked) 163 val = entry->clr_bits << 16; 164 else if (entry->clr_bits + 1) 165 val = (reg.mcr ? 166 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : 167 xe_mmio_read32(gt, reg)) & (~entry->clr_bits); 168 else 169 val = 0; 170 171 /* 172 * TODO: add selftest to validate all tables, regardless of platform: 173 * - Masked registers can't have set_bits with upper bits set 174 * - set_bits must be contained in clr_bits 175 */ 176 val |= entry->set_bits; 177 178 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); 179 180 if (entry->reg.mcr) 181 xe_gt_mcr_multicast_write(gt, reg_mcr, val); 182 else 183 xe_mmio_write32(gt, reg, val); 184 } 185 186 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 187 { 188 struct xe_reg_sr_entry *entry; 189 unsigned long reg; 190 int err; 191 192 if (xa_empty(&sr->xa)) 193 return; 194 195 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); 196 197 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); 198 if (err) 199 goto err_force_wake; 200 201 xa_for_each(&sr->xa, reg, entry) 202 apply_one_mmio(gt, entry); 203 204 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); 205 XE_WARN_ON(err); 206 207 return; 208 209 err_force_wake: 210 xe_gt_err(gt, "Failed to apply, err=%d\n", err); 211 } 212 213 void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe) 214 { 215 struct xe_reg_sr *sr = &hwe->reg_whitelist; 216 struct xe_gt *gt = hwe->gt; 217 struct xe_device *xe = gt_to_xe(gt); 218 struct xe_reg_sr_entry *entry; 219 struct drm_printer p; 220 u32 mmio_base = hwe->mmio_base; 221 unsigned long reg; 222 unsigned int slot = 0; 223 int err; 224 225 if (xa_empty(&sr->xa)) 226 return; 227 228 drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name); 229 230 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); 231 if (err) 232 goto err_force_wake; 233 234 p = drm_debug_printer(KBUILD_MODNAME); 235 xa_for_each(&sr->xa, reg, entry) { 236 if (slot == RING_MAX_NONPRIV_SLOTS) { 237 xe_gt_err(gt, 238 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n", 239 hwe->name, RING_MAX_NONPRIV_SLOTS); 240 break; 241 } 242 243 xe_reg_whitelist_print_entry(&p, 0, reg, entry); 244 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot), 245 reg | entry->set_bits); 246 slot++; 247 } 248 249 /* And clear the rest just in case of garbage */ 250 for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) { 251 u32 addr = RING_NOPID(mmio_base).addr; 252 253 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr); 254 } 255 256 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); 257 XE_WARN_ON(err); 258 259 return; 260 261 err_force_wake: 262 drm_err(&xe->drm, "Failed to apply, err=%d\n", err); 263 } 264 265 /** 266 * xe_reg_sr_dump - print all save/restore entries 267 * @sr: Save/restore entries 268 * @p: DRM printer 269 */ 270 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) 271 { 272 struct xe_reg_sr_entry *entry; 273 unsigned long reg; 274 275 if (!sr->name || xa_empty(&sr->xa)) 276 return; 277 278 drm_printf(p, "%s\n", sr->name); 279 xa_for_each(&sr->xa, reg, entry) 280 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", 281 reg, entry->clr_bits, entry->set_bits, 282 str_yes_no(entry->reg.masked), 283 str_yes_no(entry->reg.mcr)); 284 } 285