1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2017, 2019 The Linux Foundation. All rights reserved. */ 3 4 #ifndef __A6XX_GPU_H__ 5 #define __A6XX_GPU_H__ 6 7 8 #include "adreno_gpu.h" 9 #include "a6xx_enums.xml.h" 10 #include "a7xx_enums.xml.h" 11 #include "a6xx_perfcntrs.xml.h" 12 #include "a7xx_perfcntrs.xml.h" 13 #include "a6xx.xml.h" 14 15 #include "a6xx_gmu.h" 16 17 extern bool hang_debug; 18 19 struct cpu_gpu_lock { 20 uint32_t gpu_req; 21 uint32_t cpu_req; 22 uint32_t turn; 23 union { 24 /* a6xx: */ 25 struct { 26 uint16_t list_length; 27 uint16_t list_offset; 28 }; 29 /* a7xx+: */ 30 struct { 31 uint8_t ifpc_list_len; 32 uint8_t preemption_list_len; 33 uint16_t dynamic_list_len; 34 }; 35 }; 36 uint64_t regs[]; 37 }; 38 39 /** 40 * struct a6xx_info - a6xx specific information from device table 41 * 42 * @hwcg: hw clock gating register sequence 43 * @protect: CP_PROTECT settings 44 * @pwrup_reglist pwrup reglist for preemption 45 */ 46 struct a6xx_info { 47 const struct adreno_reglist *hwcg; 48 const struct adreno_protect *protect; 49 const struct adreno_reglist_list *pwrup_reglist; 50 const struct adreno_reglist_pipe_list *dyn_pwrup_reglist; 51 const struct adreno_reglist_list *ifpc_reglist; 52 const struct adreno_reglist *gbif_cx; 53 const struct adreno_reglist_pipe *nonctxt_reglist; 54 u32 max_slices; 55 u32 gmu_chipid; 56 u32 gmu_cgc_mode; 57 u32 prim_fifo_threshold; 58 const struct a6xx_bcm *bcms; 59 }; 60 61 struct a6xx_gpu { 62 struct adreno_gpu base; 63 64 struct drm_gem_object *sqe_bo; 65 uint64_t sqe_iova; 66 struct drm_gem_object *aqe_bo; 67 uint64_t aqe_iova; 68 69 struct msm_ringbuffer *cur_ring; 70 struct msm_ringbuffer *next_ring; 71 72 struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS]; 73 void *preempt[MSM_GPU_MAX_RINGS]; 74 uint64_t preempt_iova[MSM_GPU_MAX_RINGS]; 75 struct drm_gem_object *preempt_smmu_bo[MSM_GPU_MAX_RINGS]; 76 void *preempt_smmu[MSM_GPU_MAX_RINGS]; 77 uint64_t preempt_smmu_iova[MSM_GPU_MAX_RINGS]; 78 uint32_t last_seqno[MSM_GPU_MAX_RINGS]; 79 80 atomic_t preempt_state; 81 spinlock_t eval_lock; 82 struct timer_list preempt_timer; 83 84 unsigned int preempt_level; 85 bool uses_gmem; 86 bool skip_save_restore; 87 88 struct drm_gem_object *preempt_postamble_bo; 89 void *preempt_postamble_ptr; 90 uint64_t preempt_postamble_iova; 91 uint64_t preempt_postamble_len; 92 bool postamble_enabled; 93 94 struct a6xx_gmu gmu; 95 96 struct drm_gem_object *shadow_bo; 97 uint64_t shadow_iova; 98 uint32_t *shadow; 99 100 struct drm_gem_object *pwrup_reglist_bo; 101 #define PWRUP_REGLIST_SIZE (2 * PAGE_SIZE) 102 void *pwrup_reglist_ptr; 103 uint64_t pwrup_reglist_iova; 104 bool pwrup_reglist_emitted; 105 106 /* 107 * Offset of start of SEL regs appended to pwrup_reglist. This 108 * is equal to lock->dynamic_list_len if no SEL regs are appended 109 * to the end of the dynamic reglist. 110 */ 111 uint16_t dynamic_sel_reglist_offset; 112 113 bool has_whereami; 114 115 void __iomem *cx_misc_mmio; 116 void *llc_slice; 117 void *htw_llc_slice; 118 bool have_mmu500; 119 bool hung; 120 121 u32 cached_aperture; 122 spinlock_t aperture_lock; 123 124 u32 slice_mask; 125 }; 126 127 #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base) 128 129 /* 130 * In order to do lockless preemption we use a simple state machine to progress 131 * through the process. 132 * 133 * PREEMPT_NONE - no preemption in progress. Next state START. 134 * PREEMPT_START - The trigger is evaluating if preemption is possible. Next 135 * states: TRIGGERED, NONE 136 * PREEMPT_FINISH - An intermediate state before moving back to NONE. Next 137 * state: NONE. 138 * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next 139 * states: FAULTED, PENDING 140 * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger 141 * recovery. Next state: N/A 142 * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is 143 * checking the success of the operation. Next state: FAULTED, NONE. 144 */ 145 146 enum a6xx_preempt_state { 147 PREEMPT_NONE = 0, 148 PREEMPT_START, 149 PREEMPT_FINISH, 150 PREEMPT_TRIGGERED, 151 PREEMPT_FAULTED, 152 PREEMPT_PENDING, 153 }; 154 155 /* 156 * struct a6xx_preempt_record is a shared buffer between the microcode and the 157 * CPU to store the state for preemption. The record itself is much larger 158 * (2112k) but most of that is used by the CP for storage. 159 * 160 * There is a preemption record assigned per ringbuffer. When the CPU triggers a 161 * preemption, it fills out the record with the useful information (wptr, ring 162 * base, etc) and the microcode uses that information to set up the CP following 163 * the preemption. When a ring is switched out, the CP will save the ringbuffer 164 * state back to the record. In this way, once the records are properly set up 165 * the CPU can quickly switch back and forth between ringbuffers by only 166 * updating a few registers (often only the wptr). 167 * 168 * These are the CPU aware registers in the record: 169 * @magic: Must always be 0xAE399D6EUL 170 * @info: Type of the record - written 0 by the CPU, updated by the CP 171 * @errno: preemption error record 172 * @data: Data field in YIELD and SET_MARKER packets, Written and used by CP 173 * @cntl: Value of RB_CNTL written by CPU, save/restored by CP 174 * @rptr: Value of RB_RPTR written by CPU, save/restored by CP 175 * @wptr: Value of RB_WPTR written by CPU, save/restored by CP 176 * @_pad: Reserved/padding 177 * @rptr_addr: Value of RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 178 * @rbase: Value of RB_BASE written by CPU, save/restored by CP 179 * @counter: GPU address of the storage area for the preemption counters 180 * @bv_rptr_addr: Value of BV_RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 181 */ 182 struct a6xx_preempt_record { 183 u32 magic; 184 u32 info; 185 u32 errno; 186 u32 data; 187 u32 cntl; 188 u32 rptr; 189 u32 wptr; 190 u32 _pad; 191 u64 rptr_addr; 192 u64 rbase; 193 u64 counter; 194 u64 bv_rptr_addr; 195 }; 196 197 #define A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6EUL 198 199 #define PREEMPT_SMMU_INFO_SIZE 4096 200 201 #define PREEMPT_RECORD_SIZE(adreno_gpu) \ 202 ((adreno_gpu->info->preempt_record_size) == 0 ? \ 203 4192 * SZ_1K : (adreno_gpu->info->preempt_record_size)) 204 205 /* 206 * The preemption counter block is a storage area for the value of the 207 * preemption counters that are saved immediately before context switch. We 208 * append it on to the end of the allocation for the preemption record. 209 */ 210 #define A6XX_PREEMPT_COUNTER_SIZE (16 * 4) 211 212 struct a7xx_cp_smmu_info { 213 u32 magic; 214 u32 _pad4; 215 u64 ttbr0; 216 u32 asid; 217 u32 context_idr; 218 u32 context_bank; 219 }; 220 221 #define GEN7_CP_SMMU_INFO_MAGIC 0x241350d5UL 222 223 /* 224 * Given a register and a count, return a value to program into 225 * REG_CP_PROTECT_REG(n) - this will block both reads and writes for 226 * _len + 1 registers starting at _reg. 227 */ 228 #define A6XX_PROTECT_NORDWR(_reg, _len) \ 229 ((1 << 31) | \ 230 (((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) 231 232 /* 233 * Same as above, but allow reads over the range. For areas of mixed use (such 234 * as performance counters) this allows us to protect a much larger range with a 235 * single register 236 */ 237 #define A6XX_PROTECT_RDONLY(_reg, _len) \ 238 ((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) 239 240 extern const struct adreno_gpu_funcs a6xx_gpu_funcs; 241 extern const struct adreno_gpu_funcs a6xx_gmuwrapper_funcs; 242 extern const struct adreno_gpu_funcs a7xx_gpu_funcs; 243 extern const struct adreno_gpu_funcs a8xx_gpu_funcs; 244 245 static inline bool a6xx_has_gbif(struct adreno_gpu *gpu) 246 { 247 if(adreno_is_a630(gpu)) 248 return false; 249 250 return true; 251 } 252 253 static inline void a6xx_cx_misc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or) 254 { 255 return msm_rmw(a6xx_gpu->cx_misc_mmio + (reg << 2), mask, or); 256 } 257 258 static inline u32 a6xx_cx_misc_read(struct a6xx_gpu *a6xx_gpu, u32 reg) 259 { 260 return readl(a6xx_gpu->cx_misc_mmio + (reg << 2)); 261 } 262 263 static inline void a6xx_cx_misc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value) 264 { 265 writel(value, a6xx_gpu->cx_misc_mmio + (reg << 2)); 266 } 267 268 #define shadowptr(_a6xx_gpu, _ring) ((_a6xx_gpu)->shadow_iova + \ 269 ((_ring)->id * sizeof(uint32_t))) 270 271 int a6xx_gmu_resume(struct a6xx_gpu *gpu); 272 int a6xx_gmu_stop(struct a6xx_gpu *gpu); 273 274 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu); 275 276 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu); 277 278 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); 279 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); 280 281 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 282 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 283 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu); 284 void a6xx_gmu_sysprof_setup(struct msm_gpu *gpu, bool force_on); 285 286 void a6xx_preempt_init(struct msm_gpu *gpu); 287 void a6xx_preempt_hw_init(struct msm_gpu *gpu); 288 void a6xx_preempt_trigger(struct msm_gpu *gpu); 289 void a6xx_preempt_irq(struct msm_gpu *gpu); 290 void a6xx_preempt_fini(struct msm_gpu *gpu); 291 void a6xx_emit_set_pseudo_reg(struct msm_ringbuffer *ring, 292 struct a6xx_gpu *a6xx_gpu, struct msm_gpu_submitqueue *queue); 293 int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu, 294 struct msm_gpu_submitqueue *queue); 295 void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu, 296 struct msm_gpu_submitqueue *queue); 297 298 /* Return true if we are in a preempt state */ 299 static inline bool a6xx_in_preempt(struct a6xx_gpu *a6xx_gpu) 300 { 301 /* 302 * Make sure the read to preempt_state is ordered with respect to reads 303 * of other variables before ... 304 */ 305 smp_rmb(); 306 307 int preempt_state = atomic_read(&a6xx_gpu->preempt_state); 308 309 /* ... and after. */ 310 smp_rmb(); 311 312 return !(preempt_state == PREEMPT_NONE || 313 preempt_state == PREEMPT_FINISH); 314 } 315 316 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, 317 bool suspended); 318 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu); 319 320 void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 321 struct drm_printer *p); 322 323 struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu); 324 int a6xx_gpu_state_put(struct msm_gpu_state *state); 325 326 void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off); 327 void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert); 328 int a6xx_fenced_write(struct a6xx_gpu *gpu, u32 offset, u64 value, u32 mask, bool is_64b); 329 void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring); 330 void a6xx_flush_yield(struct msm_gpu *gpu, struct msm_ringbuffer *ring); 331 int a6xx_zap_shader_init(struct msm_gpu *gpu); 332 333 void a8xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off); 334 int a8xx_fault_handler(void *arg, unsigned long iova, int flags, void *data); 335 void a8xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring); 336 u64 a8xx_gmu_get_timestamp(struct msm_gpu *gpu); 337 u64 a8xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate); 338 int a8xx_gpu_feature_probe(struct msm_gpu *gpu); 339 void a8xx_gpu_get_slice_info(struct msm_gpu *gpu); 340 int a8xx_hw_init(struct msm_gpu *gpu); 341 irqreturn_t a8xx_irq(struct msm_gpu *gpu); 342 void a8xx_llc_activate(struct a6xx_gpu *a6xx_gpu); 343 void a8xx_preempt_hw_init(struct msm_gpu *gpu); 344 void a8xx_preempt_trigger(struct msm_gpu *gpu); 345 void a8xx_preempt_irq(struct msm_gpu *gpu); 346 bool a8xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring); 347 void a8xx_perfcntr_flush(struct msm_gpu *gpu); 348 void a8xx_recover(struct msm_gpu *gpu); 349 #endif /* __A6XX_GPU_H__ */ 350