1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <kunit/visibility.h> 9 #include <linux/align.h> 10 #include <linux/string_helpers.h> 11 #include <linux/xarray.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "regs/xe_engine_regs.h" 17 #include "regs/xe_gt_regs.h" 18 #include "xe_device.h" 19 #include "xe_device_types.h" 20 #include "xe_force_wake.h" 21 #include "xe_gt.h" 22 #include "xe_gt_mcr.h" 23 #include "xe_gt_printk.h" 24 #include "xe_hw_engine_types.h" 25 #include "xe_macros.h" 26 #include "xe_mmio.h" 27 #include "xe_rtp_types.h" 28 29 static void reg_sr_fini(struct drm_device *drm, void *arg) 30 { 31 struct xe_reg_sr *sr = arg; 32 struct xe_reg_sr_entry *entry; 33 unsigned long reg; 34 35 xa_for_each(&sr->xa, reg, entry) 36 kfree(entry); 37 38 xa_destroy(&sr->xa); 39 } 40 41 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 42 { 43 xa_init(&sr->xa); 44 sr->name = name; 45 46 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 47 } 48 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); 49 50 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 51 const struct xe_reg_sr_entry *e2) 52 { 53 /* 54 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 55 * when operating in the same register 56 */ 57 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 58 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 59 return false; 60 61 if (e1->reg.raw != e2->reg.raw) 62 return false; 63 64 return true; 65 } 66 67 static void reg_sr_inc_error(struct xe_reg_sr *sr) 68 { 69 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 70 sr->errors++; 71 #endif 72 } 73 74 int xe_reg_sr_add(struct xe_reg_sr *sr, 75 const struct xe_reg_sr_entry *e, 76 struct xe_gt *gt) 77 { 78 unsigned long idx = e->reg.addr; 79 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 80 int ret; 81 82 if (pentry) { 83 if (!compatible_entries(pentry, e)) { 84 ret = -EINVAL; 85 goto fail; 86 } 87 88 pentry->clr_bits |= e->clr_bits; 89 pentry->set_bits |= e->set_bits; 90 pentry->read_mask |= e->read_mask; 91 92 return 0; 93 } 94 95 pentry = kmalloc(sizeof(*pentry), GFP_KERNEL); 96 if (!pentry) { 97 ret = -ENOMEM; 98 goto fail; 99 } 100 101 *pentry = *e; 102 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 103 if (ret) 104 goto fail; 105 106 return 0; 107 108 fail: 109 xe_gt_err(gt, 110 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", 111 idx, e->clr_bits, e->set_bits, 112 str_yes_no(e->reg.masked), 113 str_yes_no(e->reg.mcr), 114 ret); 115 reg_sr_inc_error(sr); 116 117 return ret; 118 } 119 120 /* 121 * Convert back from encoded value to type-safe, only to be used when reg.mcr 122 * is true 123 */ 124 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) 125 { 126 return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; 127 } 128 129 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) 130 { 131 struct xe_reg reg = entry->reg; 132 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); 133 u32 val; 134 135 /* 136 * If this is a masked register, need to set the upper 16 bits. 137 * Set them to clr_bits since that is always a superset of the bits 138 * being modified. 139 * 140 * When it's not masked, we have to read it from hardware, unless we are 141 * supposed to set all bits. 142 */ 143 if (reg.masked) 144 val = entry->clr_bits << 16; 145 else if (entry->clr_bits + 1) 146 val = (reg.mcr ? 147 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : 148 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); 149 else 150 val = 0; 151 152 /* 153 * TODO: add selftest to validate all tables, regardless of platform: 154 * - Masked registers can't have set_bits with upper bits set 155 * - set_bits must be contained in clr_bits 156 */ 157 val |= entry->set_bits; 158 159 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); 160 161 if (entry->reg.mcr) 162 xe_gt_mcr_multicast_write(gt, reg_mcr, val); 163 else 164 xe_mmio_write32(>->mmio, reg, val); 165 } 166 167 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 168 { 169 struct xe_reg_sr_entry *entry; 170 unsigned long reg; 171 unsigned int fw_ref; 172 173 if (xa_empty(&sr->xa)) 174 return; 175 176 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); 177 178 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 179 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 180 goto err_force_wake; 181 182 xa_for_each(&sr->xa, reg, entry) 183 apply_one_mmio(gt, entry); 184 185 xe_force_wake_put(gt_to_fw(gt), fw_ref); 186 187 return; 188 189 err_force_wake: 190 xe_force_wake_put(gt_to_fw(gt), fw_ref); 191 xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); 192 } 193 194 /** 195 * xe_reg_sr_dump - print all save/restore entries 196 * @sr: Save/restore entries 197 * @p: DRM printer 198 */ 199 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) 200 { 201 struct xe_reg_sr_entry *entry; 202 unsigned long reg; 203 204 if (!sr->name || xa_empty(&sr->xa)) 205 return; 206 207 drm_printf(p, "%s\n", sr->name); 208 xa_for_each(&sr->xa, reg, entry) 209 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", 210 reg, entry->clr_bits, entry->set_bits, 211 str_yes_no(entry->reg.masked), 212 str_yes_no(entry->reg.mcr)); 213 } 214