1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <linux/align.h> 9 #include <linux/string_helpers.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_print.h> 13 #include <drm/drm_managed.h> 14 15 #include "xe_rtp_types.h" 16 #include "xe_device_types.h" 17 #include "xe_force_wake.h" 18 #include "xe_gt.h" 19 #include "xe_gt_mcr.h" 20 #include "xe_macros.h" 21 #include "xe_mmio.h" 22 23 #include "gt/intel_engine_regs.h" 24 #include "gt/intel_gt_regs.h" 25 26 #define XE_REG_SR_GROW_STEP_DEFAULT 16 27 28 static void reg_sr_fini(struct drm_device *drm, void *arg) 29 { 30 struct xe_reg_sr *sr = arg; 31 32 xa_destroy(&sr->xa); 33 kfree(sr->pool.arr); 34 memset(&sr->pool, 0, sizeof(sr->pool)); 35 } 36 37 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 38 { 39 xa_init(&sr->xa); 40 memset(&sr->pool, 0, sizeof(sr->pool)); 41 sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT; 42 sr->name = name; 43 44 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 45 } 46 47 int xe_reg_sr_dump_kv(struct xe_reg_sr *sr, 48 struct xe_reg_sr_kv **dst) 49 { 50 struct xe_reg_sr_kv *iter; 51 struct xe_reg_sr_entry *entry; 52 unsigned long idx; 53 54 if (xa_empty(&sr->xa)) { 55 *dst = NULL; 56 return 0; 57 } 58 59 *dst = kmalloc_array(sr->pool.used, sizeof(**dst), GFP_KERNEL); 60 if (!*dst) 61 return -ENOMEM; 62 63 iter = *dst; 64 xa_for_each(&sr->xa, idx, entry) { 65 iter->k = idx; 66 iter->v = *entry; 67 iter++; 68 } 69 70 return 0; 71 } 72 73 static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr) 74 { 75 if (sr->pool.used == sr->pool.allocated) { 76 struct xe_reg_sr_entry *arr; 77 78 arr = krealloc_array(sr->pool.arr, 79 ALIGN(sr->pool.allocated + 1, sr->pool.grow_step), 80 sizeof(*arr), GFP_KERNEL); 81 if (!arr) 82 return NULL; 83 84 sr->pool.arr = arr; 85 sr->pool.allocated += sr->pool.grow_step; 86 } 87 88 return &sr->pool.arr[sr->pool.used++]; 89 } 90 91 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 92 const struct xe_reg_sr_entry *e2) 93 { 94 /* 95 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 96 * when operating in the same register 97 */ 98 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 99 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 100 return false; 101 102 if (e1->masked_reg != e2->masked_reg) 103 return false; 104 105 if (e1->reg_type != e2->reg_type) 106 return false; 107 108 return true; 109 } 110 111 int xe_reg_sr_add(struct xe_reg_sr *sr, u32 reg, 112 const struct xe_reg_sr_entry *e) 113 { 114 unsigned long idx = reg; 115 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 116 int ret; 117 118 if (pentry) { 119 if (!compatible_entries(pentry, e)) { 120 ret = -EINVAL; 121 goto fail; 122 } 123 124 pentry->clr_bits |= e->clr_bits; 125 pentry->set_bits |= e->set_bits; 126 pentry->read_mask |= e->read_mask; 127 128 return 0; 129 } 130 131 pentry = alloc_entry(sr); 132 if (!pentry) { 133 ret = -ENOMEM; 134 goto fail; 135 } 136 137 *pentry = *e; 138 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 139 if (ret) 140 goto fail; 141 142 return 0; 143 144 fail: 145 DRM_ERROR("Discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s): ret=%d\n", 146 idx, e->clr_bits, e->set_bits, 147 str_yes_no(e->masked_reg), ret); 148 149 return ret; 150 } 151 152 static void apply_one_mmio(struct xe_gt *gt, u32 reg, 153 struct xe_reg_sr_entry *entry) 154 { 155 struct xe_device *xe = gt_to_xe(gt); 156 u32 val; 157 158 /* 159 * If this is a masked register, need to figure what goes on the upper 160 * 16 bits: it's either the clr_bits (when using FIELD_SET and WR) or 161 * the set_bits, when using SET. 162 * 163 * When it's not masked, we have to read it from hardware, unless we are 164 * supposed to set all bits. 165 */ 166 if (entry->masked_reg) 167 val = (entry->clr_bits ?: entry->set_bits << 16); 168 else if (entry->clr_bits + 1) 169 val = (entry->reg_type == XE_RTP_REG_MCR ? 170 xe_gt_mcr_unicast_read_any(gt, MCR_REG(reg)) : 171 xe_mmio_read32(gt, reg)) & (~entry->clr_bits); 172 else 173 val = 0; 174 175 /* 176 * TODO: add selftest to validate all tables, regardless of platform: 177 * - Masked registers can't have set_bits with upper bits set 178 * - set_bits must be contained in clr_bits 179 */ 180 val |= entry->set_bits; 181 182 drm_dbg(&xe->drm, "REG[0x%x] = 0x%08x", reg, val); 183 184 if (entry->reg_type == XE_RTP_REG_MCR) 185 xe_gt_mcr_multicast_write(gt, MCR_REG(reg), val); 186 else 187 xe_mmio_write32(gt, reg, val); 188 } 189 190 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 191 { 192 struct xe_device *xe = gt_to_xe(gt); 193 struct xe_reg_sr_entry *entry; 194 unsigned long reg; 195 int err; 196 197 drm_dbg(&xe->drm, "Applying %s save-restore MMIOs\n", sr->name); 198 199 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); 200 if (err) 201 goto err_force_wake; 202 203 xa_for_each(&sr->xa, reg, entry) 204 apply_one_mmio(gt, reg, entry); 205 206 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); 207 XE_WARN_ON(err); 208 209 return; 210 211 err_force_wake: 212 drm_err(&xe->drm, "Failed to apply, err=%d\n", err); 213 } 214 215 void xe_reg_sr_apply_whitelist(struct xe_reg_sr *sr, u32 mmio_base, 216 struct xe_gt *gt) 217 { 218 struct xe_device *xe = gt_to_xe(gt); 219 struct xe_reg_sr_entry *entry; 220 unsigned long reg; 221 unsigned int slot = 0; 222 int err; 223 224 drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name); 225 226 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); 227 if (err) 228 goto err_force_wake; 229 230 xa_for_each(&sr->xa, reg, entry) { 231 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot).reg, 232 reg | entry->set_bits); 233 slot++; 234 } 235 236 /* And clear the rest just in case of garbage */ 237 for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) 238 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot).reg, 239 RING_NOPID(mmio_base).reg); 240 241 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); 242 XE_WARN_ON(err); 243 244 return; 245 246 err_force_wake: 247 drm_err(&xe->drm, "Failed to apply, err=%d\n", err); 248 } 249