1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <kunit/visibility.h> 9 #include <linux/align.h> 10 #include <linux/string_helpers.h> 11 #include <linux/xarray.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "xe_assert.h" 17 #include "xe_device.h" 18 #include "xe_device_types.h" 19 #include "xe_force_wake.h" 20 #include "xe_gt_mcr.h" 21 #include "xe_gt_printk.h" 22 #include "xe_gt_types.h" 23 #include "xe_hw_engine_types.h" 24 #include "xe_lrc.h" 25 #include "xe_mmio.h" 26 #include "xe_rtp_types.h" 27 28 static void reg_sr_fini(struct drm_device *drm, void *arg) 29 { 30 struct xe_reg_sr *sr = arg; 31 struct xe_reg_sr_entry *entry; 32 unsigned long reg; 33 34 xa_for_each(&sr->xa, reg, entry) 35 kfree(entry); 36 37 xa_destroy(&sr->xa); 38 } 39 40 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 41 { 42 xa_init(&sr->xa); 43 sr->name = name; 44 45 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 46 } 47 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); 48 49 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 50 const struct xe_reg_sr_entry *e2) 51 { 52 /* 53 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 54 * when operating in the same register 55 */ 56 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 57 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 58 return false; 59 60 if (e1->reg.raw != e2->reg.raw) 61 return false; 62 63 return true; 64 } 65 66 static void reg_sr_inc_error(struct xe_reg_sr *sr) 67 { 68 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 69 sr->errors++; 70 #endif 71 } 72 73 static struct xe_reg sanitize_mcr(struct xe_reg_sr *sr, 74 const struct xe_reg_sr_entry *e, 75 struct xe_gt *gt) 76 { 77 struct xe_reg reg = e->reg; 78 bool is_mcr; 79 80 /* 81 * We need the gt structure to check MCR ranges. 82 */ 83 if (!gt) 84 return reg; 85 86 is_mcr = xe_gt_mcr_check_reg(gt, reg); 87 88 if (is_mcr && !reg.mcr) { 89 reg.mcr = 1; 90 xe_gt_notice(gt, "xe_reg_sr_entry using non-MCR register for address 0x%x, forcing MCR\n", 91 reg.addr); 92 reg_sr_inc_error(sr); 93 } 94 95 if (!is_mcr && reg.mcr) { 96 reg.mcr = 0; 97 xe_gt_notice(gt, "xe_reg_sr_entry using MCR register for address 0x%x, forcing non-MCR\n", 98 reg.addr); 99 reg_sr_inc_error(sr); 100 } 101 102 return reg; 103 } 104 105 int xe_reg_sr_add(struct xe_reg_sr *sr, 106 const struct xe_reg_sr_entry *e, 107 struct xe_gt *gt) 108 { 109 unsigned long idx = e->reg.addr; 110 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 111 struct xe_reg reg; 112 int ret; 113 114 reg = sanitize_mcr(sr, e, gt); 115 116 if (pentry) { 117 if (!compatible_entries(pentry, e)) { 118 ret = -EINVAL; 119 goto fail; 120 } 121 122 pentry->clr_bits |= e->clr_bits; 123 pentry->set_bits |= e->set_bits; 124 pentry->read_mask |= e->read_mask; 125 126 return 0; 127 } 128 129 pentry = kmalloc_obj(*pentry); 130 if (!pentry) { 131 ret = -ENOMEM; 132 goto fail; 133 } 134 135 *pentry = *e; 136 pentry->reg = reg; 137 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 138 if (ret) 139 goto fail_free; 140 141 return 0; 142 143 fail_free: 144 kfree(pentry); 145 fail: 146 xe_gt_err(gt, 147 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", 148 idx, e->clr_bits, e->set_bits, 149 str_yes_no(e->reg.masked), 150 str_yes_no(e->reg.mcr), 151 ret); 152 reg_sr_inc_error(sr); 153 154 return ret; 155 } 156 157 /* 158 * Convert back from encoded value to type-safe, only to be used when reg.mcr 159 * is true 160 */ 161 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) 162 { 163 return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; 164 } 165 166 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) 167 { 168 struct xe_reg reg = entry->reg; 169 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); 170 u32 val; 171 172 /* 173 * If this is a masked register, need to set the upper 16 bits. 174 * Set them to clr_bits since that is always a superset of the bits 175 * being modified. 176 * 177 * When it's not masked, we have to read it from hardware, unless we are 178 * supposed to set all bits. 179 */ 180 if (reg.masked) 181 val = entry->clr_bits << 16; 182 else if (entry->clr_bits + 1) 183 val = (reg.mcr ? 184 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : 185 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); 186 else 187 val = 0; 188 189 /* 190 * TODO: add selftest to validate all tables, regardless of platform: 191 * - Masked registers can't have set_bits with upper bits set 192 * - set_bits must be contained in clr_bits 193 */ 194 val |= entry->set_bits; 195 196 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); 197 198 if (entry->reg.mcr) 199 xe_gt_mcr_multicast_write(gt, reg_mcr, val); 200 else 201 xe_mmio_write32(>->mmio, reg, val); 202 } 203 204 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 205 { 206 struct xe_reg_sr_entry *entry; 207 unsigned long reg; 208 209 if (xa_empty(&sr->xa)) 210 return; 211 212 /* 213 * We don't process non-LRC reg_sr lists in VF, so they should have 214 * been empty in the check above. 215 */ 216 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 217 218 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); 219 220 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); 221 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { 222 xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); 223 return; 224 } 225 226 xa_for_each(&sr->xa, reg, entry) 227 apply_one_mmio(gt, entry); 228 } 229 230 /** 231 * xe_reg_sr_dump - print all save/restore entries 232 * @sr: Save/restore entries 233 * @p: DRM printer 234 */ 235 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) 236 { 237 struct xe_reg_sr_entry *entry; 238 unsigned long reg; 239 240 if (!sr->name || xa_empty(&sr->xa)) 241 return; 242 243 drm_printf(p, "%s\n", sr->name); 244 xa_for_each(&sr->xa, reg, entry) 245 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", 246 reg, entry->clr_bits, entry->set_bits, 247 str_yes_no(entry->reg.masked), 248 str_yes_no(entry->reg.mcr)); 249 } 250 251 static u32 readback_reg(struct xe_gt *gt, struct xe_reg reg) 252 { 253 struct xe_reg_mcr mcr_reg = to_xe_reg_mcr(reg); 254 255 if (reg.mcr) 256 return xe_gt_mcr_unicast_read_any(gt, mcr_reg); 257 else 258 return xe_mmio_read32(>->mmio, reg); 259 } 260 261 /** 262 * xe_reg_sr_readback_check() - Readback registers referenced in save/restore 263 * entries and check whether the programming is in place. 264 * @sr: Save/restore entries 265 * @gt: GT to read register from 266 * @p: DRM printer to report discrepancies on 267 */ 268 void xe_reg_sr_readback_check(struct xe_reg_sr *sr, 269 struct xe_gt *gt, 270 struct drm_printer *p) 271 { 272 struct xe_reg_sr_entry *entry; 273 unsigned long offset; 274 275 xa_for_each(&sr->xa, offset, entry) { 276 u32 val = readback_reg(gt, entry->reg); 277 u32 mask = entry->clr_bits | entry->set_bits; 278 279 if ((val & mask) != entry->set_bits) 280 drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n", 281 offset, mask, entry->set_bits, val & mask); 282 } 283 } 284 285 /** 286 * xe_reg_sr_lrc_check() - Check LRC for registers referenced in save/restore 287 * entries and check whether the programming is in place. 288 * @sr: Save/restore entries 289 * @gt: GT to read register from 290 * @hwe: Hardware engine type to check LRC for 291 * @p: DRM printer to report discrepancies on 292 */ 293 void xe_reg_sr_lrc_check(struct xe_reg_sr *sr, 294 struct xe_gt *gt, 295 struct xe_hw_engine *hwe, 296 struct drm_printer *p) 297 { 298 struct xe_reg_sr_entry *entry; 299 unsigned long offset; 300 301 xa_for_each(&sr->xa, offset, entry) { 302 u32 val; 303 int ret = xe_lrc_lookup_default_reg_value(gt, hwe->class, offset, &val); 304 u32 mask = entry->clr_bits | entry->set_bits; 305 306 if (ret == -ENOENT) 307 drm_printf(p, "%#8lx :: not found in LRC for %s\n", offset, hwe->name); 308 else if ((val & mask) != entry->set_bits) 309 drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n", 310 offset, mask, entry->set_bits, val & mask); 311 } 312 } 313