1dd08ebf6SMatthew Brost // SPDX-License-Identifier: MIT 2dd08ebf6SMatthew Brost /* 3dd08ebf6SMatthew Brost * Copyright © 2022 Intel Corporation 4dd08ebf6SMatthew Brost */ 5dd08ebf6SMatthew Brost 6dd08ebf6SMatthew Brost #include "xe_reg_sr.h" 7dd08ebf6SMatthew Brost 8dd08ebf6SMatthew Brost #include <linux/align.h> 9dd08ebf6SMatthew Brost #include <linux/string_helpers.h> 10dd08ebf6SMatthew Brost #include <linux/xarray.h> 11dd08ebf6SMatthew Brost 12dd08ebf6SMatthew Brost #include <drm/drm_managed.h> 13ea9f879dSLucas De Marchi #include <drm/drm_print.h> 14dd08ebf6SMatthew Brost 15*b79e8fd9SLucas De Marchi #include "regs/xe_engine_regs.h" 16dd08ebf6SMatthew Brost #include "xe_device_types.h" 17dd08ebf6SMatthew Brost #include "xe_force_wake.h" 18dd08ebf6SMatthew Brost #include "xe_gt.h" 19dd08ebf6SMatthew Brost #include "xe_gt_mcr.h" 20dd08ebf6SMatthew Brost #include "xe_macros.h" 21dd08ebf6SMatthew Brost #include "xe_mmio.h" 22ea9f879dSLucas De Marchi #include "xe_rtp_types.h" 23dd08ebf6SMatthew Brost 24dd08ebf6SMatthew Brost #include "gt/intel_gt_regs.h" 25dd08ebf6SMatthew Brost 26dd08ebf6SMatthew Brost #define XE_REG_SR_GROW_STEP_DEFAULT 16 27dd08ebf6SMatthew Brost 28dd08ebf6SMatthew Brost static void reg_sr_fini(struct drm_device *drm, void *arg) 29dd08ebf6SMatthew Brost { 30dd08ebf6SMatthew Brost struct xe_reg_sr *sr = arg; 31dd08ebf6SMatthew Brost 32dd08ebf6SMatthew Brost xa_destroy(&sr->xa); 33dd08ebf6SMatthew Brost kfree(sr->pool.arr); 34dd08ebf6SMatthew Brost memset(&sr->pool, 0, sizeof(sr->pool)); 35dd08ebf6SMatthew Brost } 36dd08ebf6SMatthew Brost 37dd08ebf6SMatthew Brost int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) 38dd08ebf6SMatthew Brost { 39dd08ebf6SMatthew Brost xa_init(&sr->xa); 40dd08ebf6SMatthew Brost memset(&sr->pool, 0, sizeof(sr->pool)); 41dd08ebf6SMatthew Brost sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT; 42dd08ebf6SMatthew Brost sr->name = name; 43dd08ebf6SMatthew Brost 44dd08ebf6SMatthew Brost return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); 45dd08ebf6SMatthew Brost } 46dd08ebf6SMatthew Brost 47dd08ebf6SMatthew Brost int xe_reg_sr_dump_kv(struct xe_reg_sr *sr, 48dd08ebf6SMatthew Brost struct xe_reg_sr_kv **dst) 49dd08ebf6SMatthew Brost { 50dd08ebf6SMatthew Brost struct xe_reg_sr_kv *iter; 51dd08ebf6SMatthew Brost struct xe_reg_sr_entry *entry; 52dd08ebf6SMatthew Brost unsigned long idx; 53dd08ebf6SMatthew Brost 54dd08ebf6SMatthew Brost if (xa_empty(&sr->xa)) { 55dd08ebf6SMatthew Brost *dst = NULL; 56dd08ebf6SMatthew Brost return 0; 57dd08ebf6SMatthew Brost } 58dd08ebf6SMatthew Brost 59dd08ebf6SMatthew Brost *dst = kmalloc_array(sr->pool.used, sizeof(**dst), GFP_KERNEL); 60dd08ebf6SMatthew Brost if (!*dst) 61dd08ebf6SMatthew Brost return -ENOMEM; 62dd08ebf6SMatthew Brost 63dd08ebf6SMatthew Brost iter = *dst; 64dd08ebf6SMatthew Brost xa_for_each(&sr->xa, idx, entry) { 65dd08ebf6SMatthew Brost iter->k = idx; 66dd08ebf6SMatthew Brost iter->v = *entry; 67dd08ebf6SMatthew Brost iter++; 68dd08ebf6SMatthew Brost } 69dd08ebf6SMatthew Brost 70dd08ebf6SMatthew Brost return 0; 71dd08ebf6SMatthew Brost } 72dd08ebf6SMatthew Brost 73dd08ebf6SMatthew Brost static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr) 74dd08ebf6SMatthew Brost { 75dd08ebf6SMatthew Brost if (sr->pool.used == sr->pool.allocated) { 76dd08ebf6SMatthew Brost struct xe_reg_sr_entry *arr; 77dd08ebf6SMatthew Brost 78dd08ebf6SMatthew Brost arr = krealloc_array(sr->pool.arr, 79dd08ebf6SMatthew Brost ALIGN(sr->pool.allocated + 1, sr->pool.grow_step), 80dd08ebf6SMatthew Brost sizeof(*arr), GFP_KERNEL); 81dd08ebf6SMatthew Brost if (!arr) 82dd08ebf6SMatthew Brost return NULL; 83dd08ebf6SMatthew Brost 84dd08ebf6SMatthew Brost sr->pool.arr = arr; 85dd08ebf6SMatthew Brost sr->pool.allocated += sr->pool.grow_step; 86dd08ebf6SMatthew Brost } 87dd08ebf6SMatthew Brost 88dd08ebf6SMatthew Brost return &sr->pool.arr[sr->pool.used++]; 89dd08ebf6SMatthew Brost } 90dd08ebf6SMatthew Brost 91dd08ebf6SMatthew Brost static bool compatible_entries(const struct xe_reg_sr_entry *e1, 92dd08ebf6SMatthew Brost const struct xe_reg_sr_entry *e2) 93dd08ebf6SMatthew Brost { 94dd08ebf6SMatthew Brost /* 95dd08ebf6SMatthew Brost * Don't allow overwriting values: clr_bits/set_bits should be disjoint 96dd08ebf6SMatthew Brost * when operating in the same register 97dd08ebf6SMatthew Brost */ 98dd08ebf6SMatthew Brost if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || 99dd08ebf6SMatthew Brost e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) 100dd08ebf6SMatthew Brost return false; 101dd08ebf6SMatthew Brost 102dd08ebf6SMatthew Brost if (e1->masked_reg != e2->masked_reg) 103dd08ebf6SMatthew Brost return false; 104dd08ebf6SMatthew Brost 105dd08ebf6SMatthew Brost if (e1->reg_type != e2->reg_type) 106dd08ebf6SMatthew Brost return false; 107dd08ebf6SMatthew Brost 108dd08ebf6SMatthew Brost return true; 109dd08ebf6SMatthew Brost } 110dd08ebf6SMatthew Brost 111dd08ebf6SMatthew Brost int xe_reg_sr_add(struct xe_reg_sr *sr, u32 reg, 112dd08ebf6SMatthew Brost const struct xe_reg_sr_entry *e) 113dd08ebf6SMatthew Brost { 114dd08ebf6SMatthew Brost unsigned long idx = reg; 115dd08ebf6SMatthew Brost struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); 116dd08ebf6SMatthew Brost int ret; 117dd08ebf6SMatthew Brost 118dd08ebf6SMatthew Brost if (pentry) { 119dd08ebf6SMatthew Brost if (!compatible_entries(pentry, e)) { 120dd08ebf6SMatthew Brost ret = -EINVAL; 121dd08ebf6SMatthew Brost goto fail; 122dd08ebf6SMatthew Brost } 123dd08ebf6SMatthew Brost 124dd08ebf6SMatthew Brost pentry->clr_bits |= e->clr_bits; 125dd08ebf6SMatthew Brost pentry->set_bits |= e->set_bits; 126dd08ebf6SMatthew Brost pentry->read_mask |= e->read_mask; 127dd08ebf6SMatthew Brost 128dd08ebf6SMatthew Brost return 0; 129dd08ebf6SMatthew Brost } 130dd08ebf6SMatthew Brost 131dd08ebf6SMatthew Brost pentry = alloc_entry(sr); 132dd08ebf6SMatthew Brost if (!pentry) { 133dd08ebf6SMatthew Brost ret = -ENOMEM; 134dd08ebf6SMatthew Brost goto fail; 135dd08ebf6SMatthew Brost } 136dd08ebf6SMatthew Brost 137dd08ebf6SMatthew Brost *pentry = *e; 138dd08ebf6SMatthew Brost ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); 139dd08ebf6SMatthew Brost if (ret) 140dd08ebf6SMatthew Brost goto fail; 141dd08ebf6SMatthew Brost 142dd08ebf6SMatthew Brost return 0; 143dd08ebf6SMatthew Brost 144dd08ebf6SMatthew Brost fail: 145dd08ebf6SMatthew Brost DRM_ERROR("Discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s): ret=%d\n", 146dd08ebf6SMatthew Brost idx, e->clr_bits, e->set_bits, 147dd08ebf6SMatthew Brost str_yes_no(e->masked_reg), ret); 148dd08ebf6SMatthew Brost 149dd08ebf6SMatthew Brost return ret; 150dd08ebf6SMatthew Brost } 151dd08ebf6SMatthew Brost 152dd08ebf6SMatthew Brost static void apply_one_mmio(struct xe_gt *gt, u32 reg, 153dd08ebf6SMatthew Brost struct xe_reg_sr_entry *entry) 154dd08ebf6SMatthew Brost { 155dd08ebf6SMatthew Brost struct xe_device *xe = gt_to_xe(gt); 156dd08ebf6SMatthew Brost u32 val; 157dd08ebf6SMatthew Brost 158dd08ebf6SMatthew Brost /* 159dd08ebf6SMatthew Brost * If this is a masked register, need to figure what goes on the upper 160dd08ebf6SMatthew Brost * 16 bits: it's either the clr_bits (when using FIELD_SET and WR) or 161dd08ebf6SMatthew Brost * the set_bits, when using SET. 162dd08ebf6SMatthew Brost * 163dd08ebf6SMatthew Brost * When it's not masked, we have to read it from hardware, unless we are 164dd08ebf6SMatthew Brost * supposed to set all bits. 165dd08ebf6SMatthew Brost */ 166dd08ebf6SMatthew Brost if (entry->masked_reg) 167dd08ebf6SMatthew Brost val = (entry->clr_bits ?: entry->set_bits << 16); 168dd08ebf6SMatthew Brost else if (entry->clr_bits + 1) 169dd08ebf6SMatthew Brost val = (entry->reg_type == XE_RTP_REG_MCR ? 170dd08ebf6SMatthew Brost xe_gt_mcr_unicast_read_any(gt, MCR_REG(reg)) : 171dd08ebf6SMatthew Brost xe_mmio_read32(gt, reg)) & (~entry->clr_bits); 172dd08ebf6SMatthew Brost else 173dd08ebf6SMatthew Brost val = 0; 174dd08ebf6SMatthew Brost 175dd08ebf6SMatthew Brost /* 176dd08ebf6SMatthew Brost * TODO: add selftest to validate all tables, regardless of platform: 177dd08ebf6SMatthew Brost * - Masked registers can't have set_bits with upper bits set 178dd08ebf6SMatthew Brost * - set_bits must be contained in clr_bits 179dd08ebf6SMatthew Brost */ 180dd08ebf6SMatthew Brost val |= entry->set_bits; 181dd08ebf6SMatthew Brost 182dd08ebf6SMatthew Brost drm_dbg(&xe->drm, "REG[0x%x] = 0x%08x", reg, val); 183dd08ebf6SMatthew Brost 184dd08ebf6SMatthew Brost if (entry->reg_type == XE_RTP_REG_MCR) 185dd08ebf6SMatthew Brost xe_gt_mcr_multicast_write(gt, MCR_REG(reg), val); 186dd08ebf6SMatthew Brost else 187dd08ebf6SMatthew Brost xe_mmio_write32(gt, reg, val); 188dd08ebf6SMatthew Brost } 189dd08ebf6SMatthew Brost 190dd08ebf6SMatthew Brost void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) 191dd08ebf6SMatthew Brost { 192dd08ebf6SMatthew Brost struct xe_device *xe = gt_to_xe(gt); 193dd08ebf6SMatthew Brost struct xe_reg_sr_entry *entry; 194dd08ebf6SMatthew Brost unsigned long reg; 195dd08ebf6SMatthew Brost int err; 196dd08ebf6SMatthew Brost 197dd08ebf6SMatthew Brost drm_dbg(&xe->drm, "Applying %s save-restore MMIOs\n", sr->name); 198dd08ebf6SMatthew Brost 199dd08ebf6SMatthew Brost err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); 200dd08ebf6SMatthew Brost if (err) 201dd08ebf6SMatthew Brost goto err_force_wake; 202dd08ebf6SMatthew Brost 203dd08ebf6SMatthew Brost xa_for_each(&sr->xa, reg, entry) 204dd08ebf6SMatthew Brost apply_one_mmio(gt, reg, entry); 205dd08ebf6SMatthew Brost 206dd08ebf6SMatthew Brost err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); 207dd08ebf6SMatthew Brost XE_WARN_ON(err); 208dd08ebf6SMatthew Brost 209dd08ebf6SMatthew Brost return; 210dd08ebf6SMatthew Brost 211dd08ebf6SMatthew Brost err_force_wake: 212dd08ebf6SMatthew Brost drm_err(&xe->drm, "Failed to apply, err=%d\n", err); 213dd08ebf6SMatthew Brost } 214dd08ebf6SMatthew Brost 215dd08ebf6SMatthew Brost void xe_reg_sr_apply_whitelist(struct xe_reg_sr *sr, u32 mmio_base, 216dd08ebf6SMatthew Brost struct xe_gt *gt) 217dd08ebf6SMatthew Brost { 218dd08ebf6SMatthew Brost struct xe_device *xe = gt_to_xe(gt); 219dd08ebf6SMatthew Brost struct xe_reg_sr_entry *entry; 220dd08ebf6SMatthew Brost unsigned long reg; 221dd08ebf6SMatthew Brost unsigned int slot = 0; 222dd08ebf6SMatthew Brost int err; 223dd08ebf6SMatthew Brost 224dd08ebf6SMatthew Brost drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name); 225dd08ebf6SMatthew Brost 226dd08ebf6SMatthew Brost err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); 227dd08ebf6SMatthew Brost if (err) 228dd08ebf6SMatthew Brost goto err_force_wake; 229dd08ebf6SMatthew Brost 230dd08ebf6SMatthew Brost xa_for_each(&sr->xa, reg, entry) { 231dd08ebf6SMatthew Brost xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot).reg, 232dd08ebf6SMatthew Brost reg | entry->set_bits); 233dd08ebf6SMatthew Brost slot++; 234dd08ebf6SMatthew Brost } 235dd08ebf6SMatthew Brost 236dd08ebf6SMatthew Brost /* And clear the rest just in case of garbage */ 237dd08ebf6SMatthew Brost for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) 238dd08ebf6SMatthew Brost xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot).reg, 239dd08ebf6SMatthew Brost RING_NOPID(mmio_base).reg); 240dd08ebf6SMatthew Brost 241dd08ebf6SMatthew Brost err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); 242dd08ebf6SMatthew Brost XE_WARN_ON(err); 243dd08ebf6SMatthew Brost 244dd08ebf6SMatthew Brost return; 245dd08ebf6SMatthew Brost 246dd08ebf6SMatthew Brost err_force_wake: 247dd08ebf6SMatthew Brost drm_err(&xe->drm, "Failed to apply, err=%d\n", err); 248dd08ebf6SMatthew Brost } 249