1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <kunit/visibility.h> 9 #include <linux/align.h> 10 #include <linux/string_helpers.h> 11 #include <linux/xarray.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "regs/xe_engine_regs.h" 17 #include "regs/xe_gt_regs.h" 18 #include "xe_device.h" 19 #include "xe_device_types.h" 20 #include "xe_force_wake.h" 21 #include "xe_gt.h" 22 #include "xe_gt_mcr.h" 23 #include "xe_gt_printk.h" 24 #include "xe_hw_engine_types.h" 25 #include "xe_macros.h" 26 #include "xe_mmio.h" 27 #include "xe_reg_whitelist.h" 28 #include "xe_rtp_types.h" 29 30 static void reg_sr_fini(struct drm_device *drm, void *arg) 31 { 32 struct xe_reg_sr *sr = arg; 33 struct xe_reg_sr_entry *entry; 34 unsigned long reg; 35 36 xa_for_each(&sr->xa, reg, entry) 37 kfree(entry); 38 39 xa_destroy(&sr->xa); 40 } 41 42 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 43 { 44 xa_init(&sr->xa); 45 sr->name = name; 46 47 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 48 } 49 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); 50 51 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 52 const struct xe_reg_sr_entry *e2) 53 { 54 /* 55 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 56 * when operating in the same register 57 */ 58 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 59 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 60 return false; 61 62 if (e1->reg.raw != e2->reg.raw) 63 return false; 64 65 return true; 66 } 67 68 static void reg_sr_inc_error(struct xe_reg_sr *sr) 69 { 70 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 71 sr->errors++; 72 #endif 73 } 74 75 int xe_reg_sr_add(struct xe_reg_sr *sr, 76 const struct xe_reg_sr_entry *e, 77 struct xe_gt *gt) 78 { 79 unsigned long idx = e->reg.addr; 80 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 81 int ret; 82 83 if (pentry) { 84 if (!compatible_entries(pentry, e)) { 85 ret = -EINVAL; 86 goto fail; 87 } 88 89 pentry->clr_bits |= e->clr_bits; 90 pentry->set_bits |= e->set_bits; 91 pentry->read_mask |= e->read_mask; 92 93 return 0; 94 } 95 96 pentry = kmalloc(sizeof(*pentry), GFP_KERNEL); 97 if (!pentry) { 98 ret = -ENOMEM; 99 goto fail; 100 } 101 102 *pentry = *e; 103 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 104 if (ret) 105 goto fail; 106 107 return 0; 108 109 fail: 110 xe_gt_err(gt, 111 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", 112 idx, e->clr_bits, e->set_bits, 113 str_yes_no(e->reg.masked), 114 str_yes_no(e->reg.mcr), 115 ret); 116 reg_sr_inc_error(sr); 117 118 return ret; 119 } 120 121 /* 122 * Convert back from encoded value to type-safe, only to be used when reg.mcr 123 * is true 124 */ 125 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) 126 { 127 return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; 128 } 129 130 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) 131 { 132 struct xe_reg reg = entry->reg; 133 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); 134 u32 val; 135 136 /* 137 * If this is a masked register, need to set the upper 16 bits. 138 * Set them to clr_bits since that is always a superset of the bits 139 * being modified. 140 * 141 * When it's not masked, we have to read it from hardware, unless we are 142 * supposed to set all bits. 143 */ 144 if (reg.masked) 145 val = entry->clr_bits << 16; 146 else if (entry->clr_bits + 1) 147 val = (reg.mcr ? 148 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : 149 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); 150 else 151 val = 0; 152 153 /* 154 * TODO: add selftest to validate all tables, regardless of platform: 155 * - Masked registers can't have set_bits with upper bits set 156 * - set_bits must be contained in clr_bits 157 */ 158 val |= entry->set_bits; 159 160 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); 161 162 if (entry->reg.mcr) 163 xe_gt_mcr_multicast_write(gt, reg_mcr, val); 164 else 165 xe_mmio_write32(>->mmio, reg, val); 166 } 167 168 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 169 { 170 struct xe_reg_sr_entry *entry; 171 unsigned long reg; 172 unsigned int fw_ref; 173 174 if (xa_empty(&sr->xa)) 175 return; 176 177 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); 178 179 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 180 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 181 goto err_force_wake; 182 183 xa_for_each(&sr->xa, reg, entry) 184 apply_one_mmio(gt, entry); 185 186 xe_force_wake_put(gt_to_fw(gt), fw_ref); 187 188 return; 189 190 err_force_wake: 191 xe_force_wake_put(gt_to_fw(gt), fw_ref); 192 xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); 193 } 194 195 void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe) 196 { 197 struct xe_reg_sr *sr = &hwe->reg_whitelist; 198 struct xe_gt *gt = hwe->gt; 199 struct xe_device *xe = gt_to_xe(gt); 200 struct xe_reg_sr_entry *entry; 201 struct drm_printer p; 202 u32 mmio_base = hwe->mmio_base; 203 unsigned long reg; 204 unsigned int slot = 0; 205 unsigned int fw_ref; 206 207 if (xa_empty(&sr->xa)) 208 return; 209 210 drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name); 211 212 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 213 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 214 goto err_force_wake; 215 216 p = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL); 217 xa_for_each(&sr->xa, reg, entry) { 218 if (slot == RING_MAX_NONPRIV_SLOTS) { 219 xe_gt_err(gt, 220 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n", 221 hwe->name, RING_MAX_NONPRIV_SLOTS); 222 break; 223 } 224 225 xe_reg_whitelist_print_entry(&p, 0, reg, entry); 226 xe_mmio_write32(>->mmio, RING_FORCE_TO_NONPRIV(mmio_base, slot), 227 reg | entry->set_bits); 228 slot++; 229 } 230 231 /* And clear the rest just in case of garbage */ 232 for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) { 233 u32 addr = RING_NOPID(mmio_base).addr; 234 235 xe_mmio_write32(>->mmio, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr); 236 } 237 238 xe_force_wake_put(gt_to_fw(gt), fw_ref); 239 240 return; 241 242 err_force_wake: 243 xe_force_wake_put(gt_to_fw(gt), fw_ref); 244 drm_err(&xe->drm, "Failed to apply, err=-ETIMEDOUT\n"); 245 } 246 247 /** 248 * xe_reg_sr_dump - print all save/restore entries 249 * @sr: Save/restore entries 250 * @p: DRM printer 251 */ 252 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) 253 { 254 struct xe_reg_sr_entry *entry; 255 unsigned long reg; 256 257 if (!sr->name || xa_empty(&sr->xa)) 258 return; 259 260 drm_printf(p, "%s\n", sr->name); 261 xa_for_each(&sr->xa, reg, entry) 262 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", 263 reg, entry->clr_bits, entry->set_bits, 264 str_yes_no(entry->reg.masked), 265 str_yes_no(entry->reg.mcr)); 266 } 267