1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_reg_sr.h" 7 8 #include <kunit/visibility.h> 9 #include <linux/align.h> 10 #include <linux/string_helpers.h> 11 #include <linux/xarray.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "xe_device.h" 17 #include "xe_device_types.h" 18 #include "xe_force_wake.h" 19 #include "xe_gt_mcr.h" 20 #include "xe_gt_printk.h" 21 #include "xe_gt_types.h" 22 #include "xe_hw_engine_types.h" 23 #include "xe_mmio.h" 24 #include "xe_rtp_types.h" 25 26 static void reg_sr_fini(struct drm_device *drm, void *arg) 27 { 28 struct xe_reg_sr *sr = arg; 29 struct xe_reg_sr_entry *entry; 30 unsigned long reg; 31 32 xa_for_each(&sr->xa, reg, entry) 33 kfree(entry); 34 35 xa_destroy(&sr->xa); 36 } 37 38 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 39 { 40 xa_init(&sr->xa); 41 sr->name = name; 42 43 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 44 } 45 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); 46 47 static bool compatible_entries(const struct xe_reg_sr_entry *e1, 48 const struct xe_reg_sr_entry *e2) 49 { 50 /* 51 * Don't allow overwriting values: clr_bits/set_bits should be disjoint 52 * when operating in the same register 53 */ 54 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 55 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 56 return false; 57 58 if (e1->reg.raw != e2->reg.raw) 59 return false; 60 61 return true; 62 } 63 64 static void reg_sr_inc_error(struct xe_reg_sr *sr) 65 { 66 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 67 sr->errors++; 68 #endif 69 } 70 71 int xe_reg_sr_add(struct xe_reg_sr *sr, 72 const struct xe_reg_sr_entry *e, 73 struct xe_gt *gt) 74 { 75 unsigned long idx = e->reg.addr; 76 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 77 int ret; 78 79 if (pentry) { 80 if (!compatible_entries(pentry, e)) { 81 ret = -EINVAL; 82 goto fail; 83 } 84 85 pentry->clr_bits |= e->clr_bits; 86 pentry->set_bits |= e->set_bits; 87 pentry->read_mask |= e->read_mask; 88 89 return 0; 90 } 91 92 pentry = kmalloc_obj(*pentry); 93 if (!pentry) { 94 ret = -ENOMEM; 95 goto fail; 96 } 97 98 *pentry = *e; 99 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 100 if (ret) 101 goto fail_free; 102 103 return 0; 104 105 fail_free: 106 kfree(pentry); 107 fail: 108 xe_gt_err(gt, 109 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", 110 idx, e->clr_bits, e->set_bits, 111 str_yes_no(e->reg.masked), 112 str_yes_no(e->reg.mcr), 113 ret); 114 reg_sr_inc_error(sr); 115 116 return ret; 117 } 118 119 /* 120 * Convert back from encoded value to type-safe, only to be used when reg.mcr 121 * is true 122 */ 123 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) 124 { 125 return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; 126 } 127 128 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) 129 { 130 struct xe_reg reg = entry->reg; 131 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); 132 u32 val; 133 134 /* 135 * If this is a masked register, need to set the upper 16 bits. 136 * Set them to clr_bits since that is always a superset of the bits 137 * being modified. 138 * 139 * When it's not masked, we have to read it from hardware, unless we are 140 * supposed to set all bits. 141 */ 142 if (reg.masked) 143 val = entry->clr_bits << 16; 144 else if (entry->clr_bits + 1) 145 val = (reg.mcr ? 146 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : 147 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); 148 else 149 val = 0; 150 151 /* 152 * TODO: add selftest to validate all tables, regardless of platform: 153 * - Masked registers can't have set_bits with upper bits set 154 * - set_bits must be contained in clr_bits 155 */ 156 val |= entry->set_bits; 157 158 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); 159 160 if (entry->reg.mcr) 161 xe_gt_mcr_multicast_write(gt, reg_mcr, val); 162 else 163 xe_mmio_write32(>->mmio, reg, val); 164 } 165 166 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 167 { 168 struct xe_reg_sr_entry *entry; 169 unsigned long reg; 170 171 if (xa_empty(&sr->xa)) 172 return; 173 174 if (IS_SRIOV_VF(gt_to_xe(gt))) 175 return; 176 177 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); 178 179 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); 180 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { 181 xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); 182 return; 183 } 184 185 xa_for_each(&sr->xa, reg, entry) 186 apply_one_mmio(gt, entry); 187 } 188 189 /** 190 * xe_reg_sr_dump - print all save/restore entries 191 * @sr: Save/restore entries 192 * @p: DRM printer 193 */ 194 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) 195 { 196 struct xe_reg_sr_entry *entry; 197 unsigned long reg; 198 199 if (!sr->name || xa_empty(&sr->xa)) 200 return; 201 202 drm_printf(p, "%s\n", sr->name); 203 xa_for_each(&sr->xa, reg, entry) 204 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", 205 reg, entry->clr_bits, entry->set_bits, 206 str_yes_no(entry->reg.masked), 207 str_yes_no(entry->reg.mcr)); 208 } 209