1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <kunit/visibility.h> 9 #include <linux/align.h> 10 #include <linux/string_helpers.h> 11 #include <linux/xarray.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "regs/xe_engine_regs.h" 17 #include "regs/xe_gt_regs.h" 18 #include "xe_device.h" 19 #include "xe_device_types.h" 20 #include "xe_force_wake.h" 21 #include "xe_gt.h" 22 #include "xe_gt_mcr.h" 23 #include "xe_gt_printk.h" 24 #include "xe_hw_engine_types.h" 25 #include "xe_macros.h" 26 #include "xe_mmio.h" 27 #include "xe_reg_whitelist.h" 28 #include "xe_rtp_types.h" 29 30 #define XE_REG_SR_GROW_STEP_DEFAULT 16 31 32 static void reg_sr_fini(struct drm_device *drm, void *arg) 33 { 34 struct xe_reg_sr *sr = arg; 35 36 xa_destroy(&sr->xa); 37 kfree(sr->pool.arr); 38 memset(&sr->pool, 0, sizeof(sr->pool)); 39 } 40 41 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 42 { 43 xa_init(&sr->xa); 44 memset(&sr->pool, 0, sizeof(sr->pool)); 45 sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT; 46 sr->name = name; 47 48 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 49 } 50 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); 51 52 static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr) 53 { 54 if (sr->pool.used == sr->pool.allocated) { 55 struct xe_reg_sr_entry *arr; 56 57 arr = krealloc_array(sr->pool.arr, 58 ALIGN(sr->pool.allocated + 1, sr->pool.grow_step), 59 sizeof(*arr), GFP_KERNEL); 60 if (!arr) 61 return NULL; 62 63 sr->pool.arr = arr; 64 sr->pool.allocated += sr->pool.grow_step; 65 } 66 67 return &sr->pool.arr[sr->pool.used++]; 68 } 69 70 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 71 const struct xe_reg_sr_entry *e2) 72 { 73 /* 74 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 75 * when operating in the same register 76 */ 77 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 78 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 79 return false; 80 81 if (e1->reg.raw != e2->reg.raw) 82 return false; 83 84 return true; 85 } 86 87 static void reg_sr_inc_error(struct xe_reg_sr *sr) 88 { 89 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 90 sr->errors++; 91 #endif 92 } 93 94 int xe_reg_sr_add(struct xe_reg_sr *sr, 95 const struct xe_reg_sr_entry *e, 96 struct xe_gt *gt) 97 { 98 unsigned long idx = e->reg.addr; 99 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 100 int ret; 101 102 if (pentry) { 103 if (!compatible_entries(pentry, e)) { 104 ret = -EINVAL; 105 goto fail; 106 } 107 108 pentry->clr_bits |= e->clr_bits; 109 pentry->set_bits |= e->set_bits; 110 pentry->read_mask |= e->read_mask; 111 112 return 0; 113 } 114 115 pentry = alloc_entry(sr); 116 if (!pentry) { 117 ret = -ENOMEM; 118 goto fail; 119 } 120 121 *pentry = *e; 122 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 123 if (ret) 124 goto fail; 125 126 return 0; 127 128 fail: 129 xe_gt_err(gt, 130 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", 131 idx, e->clr_bits, e->set_bits, 132 str_yes_no(e->reg.masked), 133 str_yes_no(e->reg.mcr), 134 ret); 135 reg_sr_inc_error(sr); 136 137 return ret; 138 } 139 140 /* 141 * Convert back from encoded value to type-safe, only to be used when reg.mcr 142 * is true 143 */ 144 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) 145 { 146 return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; 147 } 148 149 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) 150 { 151 struct xe_reg reg = entry->reg; 152 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); 153 u32 val; 154 155 /* 156 * If this is a masked register, need to set the upper 16 bits. 157 * Set them to clr_bits since that is always a superset of the bits 158 * being modified. 159 * 160 * When it's not masked, we have to read it from hardware, unless we are 161 * supposed to set all bits. 162 */ 163 if (reg.masked) 164 val = entry->clr_bits << 16; 165 else if (entry->clr_bits + 1) 166 val = (reg.mcr ? 167 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : 168 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); 169 else 170 val = 0; 171 172 /* 173 * TODO: add selftest to validate all tables, regardless of platform: 174 * - Masked registers can't have set_bits with upper bits set 175 * - set_bits must be contained in clr_bits 176 */ 177 val |= entry->set_bits; 178 179 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); 180 181 if (entry->reg.mcr) 182 xe_gt_mcr_multicast_write(gt, reg_mcr, val); 183 else 184 xe_mmio_write32(>->mmio, reg, val); 185 } 186 187 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 188 { 189 struct xe_reg_sr_entry *entry; 190 unsigned long reg; 191 unsigned int fw_ref; 192 193 if (xa_empty(&sr->xa)) 194 return; 195 196 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); 197 198 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 199 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 200 goto err_force_wake; 201 202 xa_for_each(&sr->xa, reg, entry) 203 apply_one_mmio(gt, entry); 204 205 xe_force_wake_put(gt_to_fw(gt), fw_ref); 206 207 return; 208 209 err_force_wake: 210 xe_force_wake_put(gt_to_fw(gt), fw_ref); 211 xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); 212 } 213 214 void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe) 215 { 216 struct xe_reg_sr *sr = &hwe->reg_whitelist; 217 struct xe_gt *gt = hwe->gt; 218 struct xe_device *xe = gt_to_xe(gt); 219 struct xe_reg_sr_entry *entry; 220 struct drm_printer p; 221 u32 mmio_base = hwe->mmio_base; 222 unsigned long reg; 223 unsigned int slot = 0; 224 unsigned int fw_ref; 225 226 if (xa_empty(&sr->xa)) 227 return; 228 229 drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name); 230 231 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 232 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 233 goto err_force_wake; 234 235 p = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL); 236 xa_for_each(&sr->xa, reg, entry) { 237 if (slot == RING_MAX_NONPRIV_SLOTS) { 238 xe_gt_err(gt, 239 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n", 240 hwe->name, RING_MAX_NONPRIV_SLOTS); 241 break; 242 } 243 244 xe_reg_whitelist_print_entry(&p, 0, reg, entry); 245 xe_mmio_write32(>->mmio, RING_FORCE_TO_NONPRIV(mmio_base, slot), 246 reg | entry->set_bits); 247 slot++; 248 } 249 250 /* And clear the rest just in case of garbage */ 251 for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) { 252 u32 addr = RING_NOPID(mmio_base).addr; 253 254 xe_mmio_write32(>->mmio, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr); 255 } 256 257 xe_force_wake_put(gt_to_fw(gt), fw_ref); 258 259 return; 260 261 err_force_wake: 262 xe_force_wake_put(gt_to_fw(gt), fw_ref); 263 drm_err(&xe->drm, "Failed to apply, err=-ETIMEDOUT\n"); 264 } 265 266 /** 267 * xe_reg_sr_dump - print all save/restore entries 268 * @sr: Save/restore entries 269 * @p: DRM printer 270 */ 271 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) 272 { 273 struct xe_reg_sr_entry *entry; 274 unsigned long reg; 275 276 if (!sr->name || xa_empty(&sr->xa)) 277 return; 278 279 drm_printf(p, "%s\n", sr->name); 280 xa_for_each(&sr->xa, reg, entry) 281 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", 282 reg, entry->clr_bits, entry->set_bits, 283 str_yes_no(entry->reg.masked), 284 str_yes_no(entry->reg.mcr)); 285 } 286