1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Raspberry Pi 4 */ 5 6 #include "v3d_drv.h" 7 #include "v3d_regs.h" 8 9 static const struct v3d_perf_counter_desc v3d_v42_performance_counters[] = { 10 {"FEP", "FEP-valid-primitives-no-rendered-pixels", "[FEP] Valid primitives that result in no rendered pixels, for all rendered tiles"}, 11 {"FEP", "FEP-valid-primitives-rendered-pixels", "[FEP] Valid primitives for all rendered tiles (primitives may be counted in more than one tile)"}, 12 {"FEP", "FEP-clipped-quads", "[FEP] Early-Z/Near/Far clipped quads"}, 13 {"FEP", "FEP-valid-quads", "[FEP] Valid quads"}, 14 {"TLB", "TLB-quads-not-passing-stencil-test", "[TLB] Quads with no pixels passing the stencil test"}, 15 {"TLB", "TLB-quads-not-passing-z-and-stencil-test", "[TLB] Quads with no pixels passing the Z and stencil tests"}, 16 {"TLB", "TLB-quads-passing-z-and-stencil-test", "[TLB] Quads with any pixels passing the Z and stencil tests"}, 17 {"TLB", "TLB-quads-with-zero-coverage", "[TLB] Quads with all pixels having zero coverage"}, 18 {"TLB", "TLB-quads-with-non-zero-coverage", "[TLB] Quads with any pixels having non-zero coverage"}, 19 {"TLB", "TLB-quads-written-to-color-buffer", "[TLB] Quads with valid pixels written to colour buffer"}, 20 {"PTB", "PTB-primitives-discarded-outside-viewport", "[PTB] Primitives discarded by being outside the viewport"}, 21 {"PTB", "PTB-primitives-need-clipping", "[PTB] Primitives that need clipping"}, 22 {"PTB", "PTB-primitives-discarded-reversed", "[PTB] Primitives that are discarded because they are reversed"}, 23 {"QPU", "QPU-total-idle-clk-cycles", "[QPU] Total idle clock cycles for all QPUs"}, 24 {"QPU", "QPU-total-active-clk-cycles-vertex-coord-shading", "[QPU] Total active clock cycles for all QPUs doing vertex/coordinate/user shading (counts only when QPU is not stalled)"}, 25 {"QPU", "QPU-total-active-clk-cycles-fragment-shading", "[QPU] Total active clock cycles for all QPUs doing fragment shading (counts only when QPU is not stalled)"}, 26 {"QPU", "QPU-total-clk-cycles-executing-valid-instr", "[QPU] Total clock cycles for all QPUs executing valid instructions"}, 27 {"QPU", "QPU-total-clk-cycles-waiting-TMU", "[QPU] Total clock cycles for all QPUs stalled waiting for TMUs only (counter won't increment if QPU also stalling for another reason)"}, 28 {"QPU", "QPU-total-clk-cycles-waiting-scoreboard", "[QPU] Total clock cycles for all QPUs stalled waiting for Scoreboard only (counter won't increment if QPU also stalling for another reason)"}, 29 {"QPU", "QPU-total-clk-cycles-waiting-varyings", "[QPU] Total clock cycles for all QPUs stalled waiting for Varyings only (counter won't increment if QPU also stalling for another reason)"}, 30 {"QPU", "QPU-total-instr-cache-hit", "[QPU] Total instruction cache hits for all slices"}, 31 {"QPU", "QPU-total-instr-cache-miss", "[QPU] Total instruction cache misses for all slices"}, 32 {"QPU", "QPU-total-uniform-cache-hit", "[QPU] Total uniforms cache hits for all slices"}, 33 {"QPU", "QPU-total-uniform-cache-miss", "[QPU] Total uniforms cache misses for all slices"}, 34 {"TMU", "TMU-total-text-quads-access", "[TMU] Total texture cache accesses"}, 35 {"TMU", "TMU-total-text-cache-miss", "[TMU] Total texture cache misses (number of fetches from memory/L2cache)"}, 36 {"VPM", "VPM-total-clk-cycles-VDW-stalled", "[VPM] Total clock cycles VDW is stalled waiting for VPM access"}, 37 {"VPM", "VPM-total-clk-cycles-VCD-stalled", "[VPM] Total clock cycles VCD is stalled waiting for VPM access"}, 38 {"CLE", "CLE-bin-thread-active-cycles", "[CLE] Bin thread active cycles"}, 39 {"CLE", "CLE-render-thread-active-cycles", "[CLE] Render thread active cycles"}, 40 {"L2T", "L2T-total-cache-hit", "[L2T] Total Level 2 cache hits"}, 41 {"L2T", "L2T-total-cache-miss", "[L2T] Total Level 2 cache misses"}, 42 {"CORE", "cycle-count", "[CORE] Cycle counter"}, 43 {"QPU", "QPU-total-clk-cycles-waiting-vertex-coord-shading", "[QPU] Total stalled clock cycles for all QPUs doing vertex/coordinate/user shading"}, 44 {"QPU", "QPU-total-clk-cycles-waiting-fragment-shading", "[QPU] Total stalled clock cycles for all QPUs doing fragment shading"}, 45 {"PTB", "PTB-primitives-binned", "[PTB] Total primitives binned"}, 46 {"AXI", "AXI-writes-seen-watch-0", "[AXI] Writes seen by watch 0"}, 47 {"AXI", "AXI-reads-seen-watch-0", "[AXI] Reads seen by watch 0"}, 48 {"AXI", "AXI-writes-stalled-seen-watch-0", "[AXI] Write stalls seen by watch 0"}, 49 {"AXI", "AXI-reads-stalled-seen-watch-0", "[AXI] Read stalls seen by watch 0"}, 50 {"AXI", "AXI-write-bytes-seen-watch-0", "[AXI] Total bytes written seen by watch 0"}, 51 {"AXI", "AXI-read-bytes-seen-watch-0", "[AXI] Total bytes read seen by watch 0"}, 52 {"AXI", "AXI-writes-seen-watch-1", "[AXI] Writes seen by watch 1"}, 53 {"AXI", "AXI-reads-seen-watch-1", "[AXI] Reads seen by watch 1"}, 54 {"AXI", "AXI-writes-stalled-seen-watch-1", "[AXI] Write stalls seen by watch 1"}, 55 {"AXI", "AXI-reads-stalled-seen-watch-1", "[AXI] Read stalls seen by watch 1"}, 56 {"AXI", "AXI-write-bytes-seen-watch-1", "[AXI] Total bytes written seen by watch 1"}, 57 {"AXI", "AXI-read-bytes-seen-watch-1", "[AXI] Total bytes read seen by watch 1"}, 58 {"TLB", "TLB-partial-quads-written-to-color-buffer", "[TLB] Partial quads written to the colour buffer"}, 59 {"TMU", "TMU-total-config-access", "[TMU] Total config accesses"}, 60 {"L2T", "L2T-no-id-stalled", "[L2T] No ID stall"}, 61 {"L2T", "L2T-command-queue-stalled", "[L2T] Command queue full stall"}, 62 {"L2T", "L2T-TMU-writes", "[L2T] TMU write accesses"}, 63 {"TMU", "TMU-active-cycles", "[TMU] Active cycles"}, 64 {"TMU", "TMU-stalled-cycles", "[TMU] Stalled cycles"}, 65 {"CLE", "CLE-thread-active-cycles", "[CLE] Bin or render thread active cycles"}, 66 {"L2T", "L2T-TMU-reads", "[L2T] TMU read accesses"}, 67 {"L2T", "L2T-CLE-reads", "[L2T] CLE read accesses"}, 68 {"L2T", "L2T-VCD-reads", "[L2T] VCD read accesses"}, 69 {"L2T", "L2T-TMU-config-reads", "[L2T] TMU CFG read accesses"}, 70 {"L2T", "L2T-SLC0-reads", "[L2T] SLC0 read accesses"}, 71 {"L2T", "L2T-SLC1-reads", "[L2T] SLC1 read accesses"}, 72 {"L2T", "L2T-SLC2-reads", "[L2T] SLC2 read accesses"}, 73 {"L2T", "L2T-TMU-write-miss", "[L2T] TMU write misses"}, 74 {"L2T", "L2T-TMU-read-miss", "[L2T] TMU read misses"}, 75 {"L2T", "L2T-CLE-read-miss", "[L2T] CLE read misses"}, 76 {"L2T", "L2T-VCD-read-miss", "[L2T] VCD read misses"}, 77 {"L2T", "L2T-TMU-config-read-miss", "[L2T] TMU CFG read misses"}, 78 {"L2T", "L2T-SLC0-read-miss", "[L2T] SLC0 read misses"}, 79 {"L2T", "L2T-SLC1-read-miss", "[L2T] SLC1 read misses"}, 80 {"L2T", "L2T-SLC2-read-miss", "[L2T] SLC2 read misses"}, 81 {"CORE", "core-memory-writes", "[CORE] Total memory writes"}, 82 {"L2T", "L2T-memory-writes", "[L2T] Total memory writes"}, 83 {"PTB", "PTB-memory-writes", "[PTB] Total memory writes"}, 84 {"TLB", "TLB-memory-writes", "[TLB] Total memory writes"}, 85 {"CORE", "core-memory-reads", "[CORE] Total memory reads"}, 86 {"L2T", "L2T-memory-reads", "[L2T] Total memory reads"}, 87 {"PTB", "PTB-memory-reads", "[PTB] Total memory reads"}, 88 {"PSE", "PSE-memory-reads", "[PSE] Total memory reads"}, 89 {"TLB", "TLB-memory-reads", "[TLB] Total memory reads"}, 90 {"GMP", "GMP-memory-reads", "[GMP] Total memory reads"}, 91 {"PTB", "PTB-memory-words-writes", "[PTB] Total memory words written"}, 92 {"TLB", "TLB-memory-words-writes", "[TLB] Total memory words written"}, 93 {"PSE", "PSE-memory-words-reads", "[PSE] Total memory words read"}, 94 {"TLB", "TLB-memory-words-reads", "[TLB] Total memory words read"}, 95 {"TMU", "TMU-MRU-hits", "[TMU] Total MRU hits"}, 96 {"CORE", "compute-active-cycles", "[CORE] Compute active cycles"}, 97 }; 98 99 static const struct v3d_perf_counter_desc v3d_v71_performance_counters[] = { 100 {"CORE", "cycle-count", "[CORE] Cycle counter"}, 101 {"CORE", "core-active", "[CORE] Bin/Render/Compute active cycles"}, 102 {"CLE", "CLE-bin-thread-active-cycles", "[CLE] Bin thread active cycles"}, 103 {"CLE", "CLE-render-thread-active-cycles", "[CLE] Render thread active cycles"}, 104 {"CORE", "compute-active-cycles", "[CORE] Compute active cycles"}, 105 {"FEP", "FEP-valid-primitives-no-rendered-pixels", "[FEP] Valid primitives that result in no rendered pixels, for all rendered tiles"}, 106 {"FEP", "FEP-valid-primitives-rendered-pixels", "[FEP] Valid primitives for all rendered tiles (primitives may be counted in more than one tile)"}, 107 {"FEP", "FEP-clipped-quads", "[FEP] Early-Z/Near/Far clipped quads"}, 108 {"FEP", "FEP-valid-quads", "[FEP] Valid quads"}, 109 {"TLB", "TLB-quads-not-passing-stencil-test", "[TLB] Quads with no pixels passing the stencil test"}, 110 {"TLB", "TLB-quads-not-passing-z-and-stencil-test", "[TLB] Quads with no pixels passing the Z and stencil tests"}, 111 {"TLB", "TLB-quads-passing-z-and-stencil-test", "[TLB] Quads with any pixels passing the Z and stencil tests"}, 112 {"TLB", "TLB-quads-written-to-color-buffer", "[TLB] Quads with valid pixels written to colour buffer"}, 113 {"TLB", "TLB-partial-quads-written-to-color-buffer", "[TLB] Partial quads written to the colour buffer"}, 114 {"PTB", "PTB-primitives-need-clipping", "[PTB] Primitives that need clipping"}, 115 {"PTB", "PTB-primitives-discarded-outside-viewport", "[PTB] Primitives discarded by being outside the viewport"}, 116 {"PTB", "PTB-primitives-binned", "[PTB] Total primitives binned"}, 117 {"PTB", "PTB-primitives-discarded-reversed", "[PTB] Primitives that are discarded because they are reversed"}, 118 {"QPU", "QPU-total-instr-cache-hit", "[QPU] Total instruction cache hits for all slices"}, 119 {"QPU", "QPU-total-instr-cache-miss", "[QPU] Total instruction cache misses for all slices"}, 120 {"QPU", "QPU-total-uniform-cache-hit", "[QPU] Total uniforms cache hits for all slices"}, 121 {"QPU", "QPU-total-uniform-cache-miss", "[QPU] Total uniforms cache misses for all slices"}, 122 {"TMU", "TMU-active-cycles", "[TMU] Active cycles"}, 123 {"TMU", "TMU-stalled-cycles", "[TMU] Stalled cycles"}, 124 {"TMU", "TMU-total-text-quads-access", "[TMU] Total texture cache accesses"}, 125 {"TMU", "TMU-cache-x4-active-cycles", "[TMU] Cache active cycles for x4 access"}, 126 {"TMU", "TMU-cache-x4-stalled-cycles", "[TMU] Cache stalled cycles for x4 access"}, 127 {"TMU", "TMU-total-text-quads-x4-access", "[TMU] Total texture cache x4 access"}, 128 {"L2T", "L2T-total-cache-hit", "[L2T] Total Level 2 cache hits"}, 129 {"L2T", "L2T-total-cache-miss", "[L2T] Total Level 2 cache misses"}, 130 {"L2T", "L2T-local", "[L2T] Local mode access"}, 131 {"L2T", "L2T-writeback", "[L2T] Writeback"}, 132 {"L2T", "L2T-zero", "[L2T] Zero"}, 133 {"L2T", "L2T-merge", "[L2T] Merge"}, 134 {"L2T", "L2T-fill", "[L2T] Fill"}, 135 {"L2T", "L2T-stalls-no-wid", "[L2T] Stalls because no WID available"}, 136 {"L2T", "L2T-stalls-no-rid", "[L2T] Stalls because no RID available"}, 137 {"L2T", "L2T-stalls-queue-full", "[L2T] Stalls because internal queue full"}, 138 {"L2T", "L2T-stalls-wrightback", "[L2T] Stalls because writeback in flight"}, 139 {"L2T", "L2T-stalls-mem", "[L2T] Stalls because AXI blocks read"}, 140 {"L2T", "L2T-stalls-fill", "[L2T] Stalls because fill pending for victim cache-line"}, 141 {"L2T", "L2T-hitq", "[L2T] Sent request via hit queue"}, 142 {"L2T", "L2T-hitq-full", "[L2T] Sent request via main queue because hit queue is full"}, 143 {"L2T", "L2T-stalls-read-data", "[L2T] Stalls because waiting for data from SDRAM"}, 144 {"L2T", "L2T-TMU-read-hits", "[L2T] TMU read hits"}, 145 {"L2T", "L2T-TMU-read-miss", "[L2T] TMU read misses"}, 146 {"L2T", "L2T-VCD-read-hits", "[L2T] VCD read hits"}, 147 {"L2T", "L2T-VCD-read-miss", "[L2T] VCD read misses"}, 148 {"L2T", "L2T-SLC-read-hits", "[L2T] SLC read hits (all slices)"}, 149 {"L2T", "L2T-SLC-read-miss", "[L2T] SLC read misses (all slices)"}, 150 {"AXI", "AXI-writes-seen-watch-0", "[AXI] Writes seen by watch 0"}, 151 {"AXI", "AXI-reads-seen-watch-0", "[AXI] Reads seen by watch 0"}, 152 {"AXI", "AXI-writes-stalled-seen-watch-0", "[AXI] Write stalls seen by watch 0"}, 153 {"AXI", "AXI-reads-stalled-seen-watch-0", "[AXI] Read stalls seen by watch 0"}, 154 {"AXI", "AXI-write-bytes-seen-watch-0", "[AXI] Total bytes written seen by watch 0"}, 155 {"AXI", "AXI-read-bytes-seen-watch-0", "[AXI] Total bytes read seen by watch 0"}, 156 {"AXI", "AXI-writes-seen-watch-1", "[AXI] Writes seen by watch 1"}, 157 {"AXI", "AXI-reads-seen-watch-1", "[AXI] Reads seen by watch 1"}, 158 {"AXI", "AXI-writes-stalled-seen-watch-1", "[AXI] Write stalls seen by watch 1"}, 159 {"AXI", "AXI-reads-stalled-seen-watch-1", "[AXI] Read stalls seen by watch 1"}, 160 {"AXI", "AXI-write-bytes-seen-watch-1", "[AXI] Total bytes written seen by watch 1"}, 161 {"AXI", "AXI-read-bytes-seen-watch-1", "[AXI] Total bytes read seen by watch 1"}, 162 {"CORE", "core-memory-writes", "[CORE] Total memory writes"}, 163 {"L2T", "L2T-memory-writes", "[L2T] Total memory writes"}, 164 {"PTB", "PTB-memory-writes", "[PTB] Total memory writes"}, 165 {"TLB", "TLB-memory-writes", "[TLB] Total memory writes"}, 166 {"CORE", "core-memory-reads", "[CORE] Total memory reads"}, 167 {"L2T", "L2T-memory-reads", "[L2T] Total memory reads"}, 168 {"PTB", "PTB-memory-reads", "[PTB] Total memory reads"}, 169 {"PSE", "PSE-memory-reads", "[PSE] Total memory reads"}, 170 {"TLB", "TLB-memory-reads", "[TLB] Total memory reads"}, 171 {"PTB", "PTB-memory-words-writes", "[PTB] Total memory words written"}, 172 {"TLB", "TLB-memory-words-writes", "[TLB] Total memory words written"}, 173 {"PSE", "PSE-memory-words-reads", "[PSE] Total memory words read"}, 174 {"TLB", "TLB-memory-words-reads", "[TLB] Total memory words read"}, 175 {"AXI", "AXI-read-trans", "[AXI] Read transaction count"}, 176 {"AXI", "AXI-write-trans", "[AXI] Write transaction count"}, 177 {"AXI", "AXI-read-wait-cycles", "[AXI] Read total wait cycles"}, 178 {"AXI", "AXI-write-wait-cycles", "[AXI] Write total wait cycles"}, 179 {"AXI", "AXI-max-outstanding-reads", "[AXI] Maximum outstanding read transactions"}, 180 {"AXI", "AXI-max-outstanding-writes", "[AXI] Maximum outstanding write transactions"}, 181 {"QPU", "QPU-wait-bubble", "[QPU] Pipeline bubble in qcycles due all threads waiting"}, 182 {"QPU", "QPU-ic-miss-bubble", "[QPU] Pipeline bubble in qcycles due instruction-cache miss"}, 183 {"QPU", "QPU-active", "[QPU] Executed shader instruction"}, 184 {"QPU", "QPU-total-active-clk-cycles-fragment-shading", "[QPU] Total active clock cycles for all QPUs doing fragment shading (counts only when QPU is not stalled)"}, 185 {"QPU", "QPU-stalls", "[QPU] Stalled qcycles executing shader instruction"}, 186 {"QPU", "QPU-total-clk-cycles-waiting-fragment-shading", "[QPU] Total stalled clock cycles for all QPUs doing fragment shading"}, 187 {"QPU", "QPU-stalls-TMU", "[QPU] Stalled qcycles waiting for TMU"}, 188 {"QPU", "QPU-stalls-TLB", "[QPU] Stalled qcycles waiting for TLB"}, 189 {"QPU", "QPU-stalls-VPM", "[QPU] Stalled qcycles waiting for VPM"}, 190 {"QPU", "QPU-stalls-uniforms", "[QPU] Stalled qcycles waiting for uniforms"}, 191 {"QPU", "QPU-stalls-SFU", "[QPU] Stalled qcycles waiting for SFU"}, 192 {"QPU", "QPU-stalls-other", "[QPU] Stalled qcycles waiting for any other reason (vary/W/Z)"}, 193 }; 194 195 void v3d_perfmon_init(struct v3d_dev *v3d) 196 { 197 const struct v3d_perf_counter_desc *counters = NULL; 198 unsigned int max = 0; 199 200 if (v3d->ver >= V3D_GEN_71) { 201 counters = v3d_v71_performance_counters; 202 max = ARRAY_SIZE(v3d_v71_performance_counters); 203 } else if (v3d->ver >= V3D_GEN_42) { 204 counters = v3d_v42_performance_counters; 205 max = ARRAY_SIZE(v3d_v42_performance_counters); 206 } 207 208 v3d->perfmon_info.max_counters = max; 209 v3d->perfmon_info.counters = counters; 210 } 211 212 void v3d_perfmon_get(struct v3d_perfmon *perfmon) 213 { 214 if (perfmon) 215 refcount_inc(&perfmon->refcnt); 216 } 217 218 void v3d_perfmon_put(struct v3d_perfmon *perfmon) 219 { 220 if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) 221 kfree(perfmon); 222 } 223 224 static void v3d_perfmon_hw_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) 225 { 226 u8 ncounters = perfmon->ncounters; 227 u32 mask = GENMASK(ncounters - 1, 0); 228 unsigned int i; 229 230 for (i = 0; i < ncounters; i++) { 231 u32 source = i / 4; 232 u32 channel = V3D_SET_FIELD_VER(perfmon->counters[i], V3D_PCTR_S0, 233 v3d->ver); 234 235 i++; 236 channel |= V3D_SET_FIELD_VER(i < ncounters ? perfmon->counters[i] : 0, 237 V3D_PCTR_S1, v3d->ver); 238 i++; 239 channel |= V3D_SET_FIELD_VER(i < ncounters ? perfmon->counters[i] : 0, 240 V3D_PCTR_S2, v3d->ver); 241 i++; 242 channel |= V3D_SET_FIELD_VER(i < ncounters ? perfmon->counters[i] : 0, 243 V3D_PCTR_S3, v3d->ver); 244 V3D_CORE_WRITE(0, V3D_V4_PCTR_0_SRC_X(source), channel); 245 } 246 247 V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, mask); 248 V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask); 249 V3D_CORE_WRITE(0, V3D_PCTR_0_OVERFLOW, mask); 250 } 251 252 static void v3d_perfmon_hw_capture(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) 253 { 254 u32 mask = GENMASK(perfmon->ncounters - 1, 0); 255 256 for (int i = 0; i < perfmon->ncounters; i++) 257 perfmon->values[i] += V3D_CORE_READ(0, V3D_PCTR_0_PCTRX(i)); 258 259 V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask); 260 } 261 262 static void v3d_perfmon_hw_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, 263 bool capture) 264 { 265 if (capture) 266 v3d_perfmon_hw_capture(v3d, perfmon); 267 268 V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0); 269 } 270 271 void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon) 272 { 273 guard(spinlock_irqsave)(&v3d->perfmon_state.lock); 274 275 if (!perfmon || v3d->global_perfmon) 276 return; 277 278 /* Cross-queue serialization should have drained any previous perfmon 279 * job before this one runs. 280 */ 281 if (WARN_ON_ONCE(v3d->perfmon_state.active)) 282 return; 283 284 if (!pm_runtime_get_if_active(v3d->drm.dev)) 285 return; 286 287 v3d_perfmon_hw_start(v3d, perfmon); 288 v3d->perfmon_state.active = perfmon; 289 290 v3d_pm_runtime_put(v3d); 291 } 292 293 static void v3d_perfmon_capture_locked(struct v3d_dev *v3d, 294 struct v3d_perfmon *perfmon) 295 { 296 lockdep_assert_held(&v3d->perfmon_state.lock); 297 298 if (!perfmon || perfmon != v3d->perfmon_state.active) 299 return; 300 301 if (!pm_runtime_get_if_active(v3d->drm.dev)) 302 return; 303 304 v3d_perfmon_hw_capture(v3d, perfmon); 305 v3d_pm_runtime_put(v3d); 306 } 307 308 void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, 309 bool capture) 310 { 311 lockdep_assert_held(&v3d->perfmon_state.lock); 312 313 if (!perfmon || perfmon != v3d->perfmon_state.active) 314 return; 315 316 v3d->perfmon_state.active = NULL; 317 318 /* If the device is suspended, the HW has already stopped counting. */ 319 if (!pm_runtime_get_if_active(v3d->drm.dev)) 320 return; 321 322 v3d_perfmon_hw_stop(v3d, perfmon, capture); 323 v3d_pm_runtime_put(v3d); 324 } 325 326 void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon, 327 bool capture) 328 { 329 if (!perfmon) 330 return; 331 332 guard(spinlock_irqsave)(&v3d->perfmon_state.lock); 333 v3d_perfmon_stop_locked(v3d, perfmon, capture); 334 } 335 336 void 337 v3d_perfmon_suspend(struct v3d_dev *v3d) 338 { 339 guard(spinlock_irqsave)(&v3d->perfmon_state.lock); 340 341 if (!v3d->perfmon_state.active) 342 return; 343 344 v3d_perfmon_hw_stop(v3d, v3d->perfmon_state.active, true); 345 } 346 347 void 348 v3d_perfmon_resume(struct v3d_dev *v3d) 349 { 350 guard(spinlock_irqsave)(&v3d->perfmon_state.lock); 351 352 if (!v3d->perfmon_state.active) 353 return; 354 355 v3d_perfmon_hw_start(v3d, v3d->perfmon_state.active); 356 } 357 358 struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id) 359 { 360 struct v3d_perfmon *perfmon; 361 362 xa_lock(&v3d_priv->perfmons); 363 perfmon = xa_load(&v3d_priv->perfmons, id); 364 v3d_perfmon_get(perfmon); 365 xa_unlock(&v3d_priv->perfmons); 366 367 return perfmon; 368 } 369 370 void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv) 371 { 372 xa_init_flags(&v3d_priv->perfmons, XA_FLAGS_ALLOC1); 373 } 374 375 static void v3d_perfmon_delete(struct v3d_file_priv *v3d_priv, 376 struct v3d_perfmon *perfmon) 377 { 378 struct v3d_dev *v3d = v3d_priv->v3d; 379 380 /* If the active perfmon is being destroyed, stop it first */ 381 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { 382 v3d_perfmon_stop_locked(v3d, perfmon, false); 383 384 /* If the global perfmon is being destroyed, clean it and release 385 * the reference stashed in v3d_perfmon_set_global_ioctl(). 386 */ 387 if (v3d->global_perfmon == perfmon) { 388 v3d_perfmon_put(v3d->global_perfmon); 389 v3d->global_perfmon = NULL; 390 } 391 } 392 393 v3d_perfmon_put(perfmon); 394 } 395 396 void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv) 397 { 398 struct v3d_perfmon *perfmon; 399 unsigned long id; 400 401 xa_for_each(&v3d_priv->perfmons, id, perfmon) 402 v3d_perfmon_delete(v3d_priv, perfmon); 403 404 xa_destroy(&v3d_priv->perfmons); 405 } 406 407 int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data, 408 struct drm_file *file_priv) 409 { 410 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 411 struct drm_v3d_perfmon_create *req = data; 412 struct v3d_dev *v3d = v3d_priv->v3d; 413 struct v3d_perfmon *perfmon; 414 unsigned int i; 415 int ret; 416 u32 id; 417 418 /* Number of monitored counters cannot exceed HW limits. */ 419 if (req->ncounters > DRM_V3D_MAX_PERF_COUNTERS || 420 !req->ncounters) 421 return -EINVAL; 422 423 /* Make sure all counters are valid. */ 424 for (i = 0; i < req->ncounters; i++) { 425 if (req->counters[i] >= v3d->perfmon_info.max_counters) 426 return -EINVAL; 427 } 428 429 perfmon = kzalloc_flex(*perfmon, values, req->ncounters); 430 if (!perfmon) 431 return -ENOMEM; 432 433 for (i = 0; i < req->ncounters; i++) 434 perfmon->counters[i] = req->counters[i]; 435 436 perfmon->ncounters = req->ncounters; 437 438 refcount_set(&perfmon->refcnt, 1); 439 440 ret = xa_alloc(&v3d_priv->perfmons, &id, perfmon, xa_limit_32b, 441 GFP_KERNEL); 442 if (ret < 0) { 443 kfree(perfmon); 444 return ret; 445 } 446 447 req->id = id; 448 449 return 0; 450 } 451 452 int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data, 453 struct drm_file *file_priv) 454 { 455 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 456 struct drm_v3d_perfmon_destroy *req = data; 457 struct v3d_perfmon *perfmon; 458 459 perfmon = xa_erase(&v3d_priv->perfmons, req->id); 460 if (!perfmon) 461 return -EINVAL; 462 463 v3d_perfmon_delete(v3d_priv, perfmon); 464 465 return 0; 466 } 467 468 int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data, 469 struct drm_file *file_priv) 470 { 471 struct v3d_dev *v3d = to_v3d_dev(dev); 472 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 473 struct drm_v3d_perfmon_get_values *req = data; 474 u64 values[DRM_V3D_MAX_PERF_COUNTERS]; 475 struct v3d_perfmon *perfmon; 476 size_t size; 477 int ret = 0; 478 479 if (req->pad != 0) 480 return -EINVAL; 481 482 perfmon = v3d_perfmon_find(v3d_priv, req->id); 483 if (!perfmon) 484 return -EINVAL; 485 486 size = perfmon->ncounters * sizeof(u64); 487 488 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { 489 v3d_perfmon_capture_locked(v3d, perfmon); 490 memcpy(values, perfmon->values, size); 491 } 492 493 if (copy_to_user(u64_to_user_ptr(req->values_ptr), values, size)) 494 ret = -EFAULT; 495 496 v3d_perfmon_put(perfmon); 497 498 return ret; 499 } 500 501 int v3d_perfmon_get_counter_ioctl(struct drm_device *dev, void *data, 502 struct drm_file *file_priv) 503 { 504 struct drm_v3d_perfmon_get_counter *req = data; 505 struct v3d_dev *v3d = to_v3d_dev(dev); 506 const struct v3d_perf_counter_desc *counter; 507 508 for (int i = 0; i < ARRAY_SIZE(req->reserved); i++) { 509 if (req->reserved[i] != 0) 510 return -EINVAL; 511 } 512 513 if (!v3d->perfmon_info.max_counters) 514 return -EOPNOTSUPP; 515 516 /* Make sure that the counter ID is valid */ 517 if (req->counter >= v3d->perfmon_info.max_counters) 518 return -EINVAL; 519 520 counter = &v3d->perfmon_info.counters[req->counter]; 521 522 strscpy(req->name, counter->name, sizeof(req->name)); 523 strscpy(req->category, counter->category, sizeof(req->category)); 524 strscpy(req->description, counter->description, sizeof(req->description)); 525 526 return 0; 527 } 528 529 int v3d_perfmon_set_global_ioctl(struct drm_device *dev, void *data, 530 struct drm_file *file_priv) 531 { 532 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 533 struct drm_v3d_perfmon_set_global *req = data; 534 struct v3d_dev *v3d = to_v3d_dev(dev); 535 struct v3d_perfmon *perfmon; 536 537 if (req->flags & ~DRM_V3D_PERFMON_CLEAR_GLOBAL) 538 return -EINVAL; 539 540 perfmon = v3d_perfmon_find(v3d_priv, req->id); 541 if (!perfmon) 542 return -EINVAL; 543 544 /* If the request is to clear the global performance monitor */ 545 if (req->flags & DRM_V3D_PERFMON_CLEAR_GLOBAL) { 546 struct v3d_perfmon *old; 547 548 /* DRM_V3D_PERFMON_CLEAR_GLOBAL doesn't check if 549 * v3d->global_perfmon == perfmon. Therefore, there 550 * is no need to keep perfmon's reference. 551 */ 552 v3d_perfmon_put(perfmon); 553 554 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { 555 old = v3d->global_perfmon; 556 if (!old) 557 return -EINVAL; 558 559 v3d_perfmon_stop_locked(v3d, old, true); 560 v3d->global_perfmon = NULL; 561 } 562 563 v3d_perfmon_put(old); 564 565 return 0; 566 } 567 568 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) { 569 if (v3d->perfmon_state.active || v3d->global_perfmon) { 570 v3d_perfmon_put(perfmon); 571 return -EBUSY; 572 } 573 574 v3d->global_perfmon = perfmon; 575 v3d->perfmon_state.active = perfmon; 576 577 /* If the device is suspended, v3d_perfmon_resume() will 578 * program the HW on the next resume. 579 */ 580 if (pm_runtime_get_if_active(v3d->drm.dev)) { 581 v3d_perfmon_hw_start(v3d, perfmon); 582 v3d_pm_runtime_put(v3d); 583 } 584 } 585 586 return 0; 587 } 588