1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* Copyright 2026 Arm, Ltd. */ 3 /* Based on v3d_perfmon.c, Copyright (C) 2021 Raspberry Pi */ 4 5 #include <linux/device.h> 6 #include <linux/errno.h> 7 #include <linux/pm_runtime.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/uaccess.h> 11 12 #include <drm/drm_file.h> 13 #include <drm/drm_ioctl.h> 14 15 #include <uapi/drm/ethosu_accel.h> 16 17 #include "ethosu_drv.h" 18 #include "ethosu_device.h" 19 20 void ethosu_perfmon_get(struct ethosu_perfmon *perfmon) 21 { 22 if (perfmon) 23 refcount_inc(&perfmon->refcnt); 24 } 25 26 void ethosu_perfmon_put(struct ethosu_perfmon *perfmon) 27 { 28 if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) 29 kfree(perfmon); 30 } 31 32 void ethosu_perfmon_start(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon) 33 { 34 unsigned int i; 35 u8 ncounters; 36 u32 mask; 37 38 lockdep_assert_held(ðosu->perfmon_state.lock); 39 40 if (WARN_ON_ONCE(!perfmon || ethosu->perfmon_state.active)) 41 return; 42 43 writel_relaxed(PMCR_CNT_EN, ethosu->pmu_regs + NPU_REG_PMCR); 44 writel_relaxed(PMU_EV_TYPE_CYCLES, ethosu->pmu_regs + NPU_REG_PMCCNTR_CFG); 45 46 mask = 0x80000000; 47 ncounters = perfmon->ncounters - 1; 48 if (ncounters) 49 mask |= GENMASK(ncounters - 1, 0); 50 51 for (i = 0; i < ncounters; i++) 52 writel_relaxed(perfmon->counters[i], ethosu->pmu_regs + NPU_REG_PMU_EVTYPER(i)); 53 54 writel_relaxed(mask, ethosu->pmu_regs + NPU_REG_PMCNTENSET); 55 writel_relaxed(PMCR_CNT_EN | PMCR_EVENT_CNT_RST | PMCR_CYCLE_CNT_RST, 56 ethosu->pmu_regs + NPU_REG_PMCR); 57 ethosu->perfmon_state.active = perfmon; 58 } 59 60 void ethosu_perfmon_stop_locked(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon, 61 bool capture) 62 { 63 unsigned int i; 64 u8 ncounters; 65 u32 mask; 66 67 lockdep_assert_held(ðosu->perfmon_state.lock); 68 69 if (!perfmon || perfmon != ethosu->perfmon_state.active) 70 return; 71 72 ncounters = perfmon->ncounters - 1; 73 74 if (!pm_runtime_get_if_active(ethosu->base.dev)) { 75 ethosu->perfmon_state.active = NULL; 76 return; 77 } 78 79 if (capture) { 80 for (i = 0; i < ncounters; i++) 81 perfmon->values[i] += readl_relaxed(ethosu->pmu_regs + NPU_REG_PMU_EVCNTR(i)); 82 83 perfmon->values[ncounters] += 84 readl_relaxed(ethosu->pmu_regs + NPU_REG_PMCCNTR_LO) | 85 (u64)readl_relaxed(ethosu->pmu_regs + NPU_REG_PMCCNTR_HI) << 32; 86 } 87 88 mask = 0x80000000; 89 if (ncounters) 90 mask |= GENMASK(ncounters - 1, 0); 91 writel_relaxed(mask, ethosu->pmu_regs + NPU_REG_PMCNTENCLR); 92 93 writel_relaxed(0, ethosu->pmu_regs + NPU_REG_PMCR); 94 ethosu->perfmon_state.active = NULL; 95 96 pm_runtime_put(ethosu->base.dev); 97 } 98 99 void ethosu_perfmon_stop(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon, 100 bool capture) 101 { 102 if (!perfmon) 103 return; 104 105 guard(mutex)(ðosu->perfmon_state.lock); 106 ethosu_perfmon_stop_locked(ethosu, perfmon, capture); 107 } 108 109 struct ethosu_perfmon *ethosu_perfmon_find(struct ethosu_file_priv *ethosu_priv, int id) 110 { 111 struct ethosu_perfmon *perfmon; 112 113 xa_lock(ðosu_priv->perfmons); 114 perfmon = xa_load(ðosu_priv->perfmons, id); 115 ethosu_perfmon_get(perfmon); 116 xa_unlock(ðosu_priv->perfmons); 117 118 return perfmon; 119 } 120 121 void ethosu_perfmon_open_file(struct ethosu_file_priv *ethosu_priv) 122 { 123 xa_init_flags(ðosu_priv->perfmons, XA_FLAGS_ALLOC1); 124 } 125 126 static void ethosu_perfmon_delete(struct ethosu_file_priv *ethosu_priv, 127 struct ethosu_perfmon *perfmon) 128 { 129 struct ethosu_device *ethosu = ethosu_priv->edev; 130 131 /* If the active perfmon is being destroyed, stop it first */ 132 scoped_guard(mutex, ðosu->perfmon_state.lock) { 133 /* If the global perfmon is being destroyed, set it to NULL */ 134 if (ethosu->global_perfmon == perfmon) { 135 ethosu->global_perfmon = NULL; 136 ethosu_perfmon_put(perfmon); 137 } 138 139 ethosu_perfmon_stop_locked(ethosu, perfmon, false); 140 } 141 142 ethosu_perfmon_put(perfmon); 143 } 144 145 void ethosu_perfmon_close_file(struct ethosu_file_priv *ethosu_priv) 146 { 147 struct ethosu_perfmon *perfmon; 148 unsigned long id; 149 150 xa_for_each(ðosu_priv->perfmons, id, perfmon) 151 ethosu_perfmon_delete(ethosu_priv, perfmon); 152 153 xa_destroy(ðosu_priv->perfmons); 154 } 155 156 int ethosu_ioctl_perfmon_create(struct drm_device *dev, void *data, 157 struct drm_file *file_priv) 158 { 159 struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv; 160 struct drm_ethosu_perfmon_create *req = data; 161 struct ethosu_device *ethosu = to_ethosu_device(dev); 162 struct ethosu_perfmon *perfmon; 163 unsigned int i, event_max; 164 int ret; 165 u32 id; 166 167 /* Number of monitored counters cannot exceed HW limits. */ 168 if (req->ncounters > ethosu->npu_info.pmu_counters) 169 return -EINVAL; 170 171 /* Make sure all counters are valid. */ 172 event_max = ethosu_is_u65(ethosu) ? 433 : 671; 173 for (i = 0; i < req->ncounters; i++) { 174 if (req->counters[i] > event_max) 175 return -EINVAL; 176 } 177 178 /* Add 1 more counter for cycle counter */ 179 req->ncounters++; 180 181 perfmon = kzalloc_flex(*perfmon, values, req->ncounters); 182 if (!perfmon) 183 return -ENOMEM; 184 185 for (i = 0; i < req->ncounters - 1; i++) 186 perfmon->counters[i] = req->counters[i]; 187 188 perfmon->ncounters = req->ncounters; 189 190 refcount_set(&perfmon->refcnt, 1); 191 192 ret = xa_alloc(ðosu_priv->perfmons, &id, perfmon, xa_limit_32b, 193 GFP_KERNEL); 194 195 if (ret < 0) { 196 kfree(perfmon); 197 return ret; 198 } 199 200 req->id = id; 201 202 return 0; 203 } 204 205 int ethosu_ioctl_perfmon_destroy(struct drm_device *dev, void *data, 206 struct drm_file *file_priv) 207 { 208 struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv; 209 struct drm_ethosu_perfmon_destroy *req = data; 210 struct ethosu_perfmon *perfmon; 211 212 perfmon = xa_erase(ðosu_priv->perfmons, req->id); 213 if (!perfmon) 214 return -EINVAL; 215 216 ethosu_perfmon_delete(ethosu_priv, perfmon); 217 218 return 0; 219 } 220 221 int ethosu_ioctl_perfmon_get_values(struct drm_device *dev, void *data, 222 struct drm_file *file_priv) 223 { 224 struct ethosu_device *ethosu = to_ethosu_device(dev); 225 struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv; 226 struct drm_ethosu_perfmon_get_values *req = data; 227 struct ethosu_perfmon *perfmon; 228 int ret = 0; 229 230 if (req->pad != 0) 231 return -EINVAL; 232 233 perfmon = ethosu_perfmon_find(ethosu_priv, req->id); 234 if (!perfmon) 235 return -EINVAL; 236 237 ret = pm_runtime_resume_and_get(dev->dev); 238 if (ret) { 239 ethosu_perfmon_put(perfmon); 240 return ret; 241 } 242 ethosu_perfmon_stop(ethosu, perfmon, true); 243 244 pm_runtime_put_autosuspend(dev->dev); 245 246 if (copy_to_user(u64_to_user_ptr(req->values_ptr), perfmon->values, 247 perfmon->ncounters * sizeof(u64))) 248 ret = -EFAULT; 249 250 ethosu_perfmon_put(perfmon); 251 252 return ret; 253 } 254 255 int ethosu_ioctl_perfmon_set_global(struct drm_device *dev, void *data, 256 struct drm_file *file_priv) 257 { 258 struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv; 259 struct drm_ethosu_perfmon_set_global *req = data; 260 struct ethosu_device *ethosu = to_ethosu_device(dev); 261 struct ethosu_perfmon *perfmon; 262 263 if (req->flags & ~DRM_ETHOSU_PERFMON_CLEAR_GLOBAL) 264 return -EINVAL; 265 266 perfmon = ethosu_perfmon_find(ethosu_priv, req->id); 267 if (!perfmon) 268 return -EINVAL; 269 270 /* If the request is to clear the global performance monitor */ 271 if (req->flags & DRM_ETHOSU_PERFMON_CLEAR_GLOBAL) { 272 struct ethosu_perfmon *old; 273 274 scoped_guard(mutex, ðosu->perfmon_state.lock) { 275 old = ethosu->global_perfmon; 276 if (!old) { 277 ethosu_perfmon_put(perfmon); 278 return -EINVAL; 279 } 280 281 ethosu->global_perfmon = NULL; 282 ethosu_perfmon_stop_locked(ethosu, old, true); 283 } 284 285 ethosu_perfmon_put(old); 286 ethosu_perfmon_put(perfmon); 287 288 return 0; 289 } 290 291 scoped_guard(mutex, ðosu->perfmon_state.lock) { 292 if (ethosu->perfmon_state.active || ethosu->global_perfmon) { 293 ethosu_perfmon_put(perfmon); 294 return -EBUSY; 295 } 296 297 ethosu->global_perfmon = perfmon; 298 } 299 300 return 0; 301 } 302