1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2017, 2019 The Linux Foundation. All rights reserved. */ 3 4 #ifndef __A6XX_GPU_H__ 5 #define __A6XX_GPU_H__ 6 7 8 #include "adreno_gpu.h" 9 #include "a6xx_enums.xml.h" 10 #include "a7xx_enums.xml.h" 11 #include "a6xx_perfcntrs.xml.h" 12 #include "a7xx_perfcntrs.xml.h" 13 #include "a6xx.xml.h" 14 15 #include "a6xx_gmu.h" 16 17 extern bool hang_debug; 18 19 struct cpu_gpu_lock { 20 uint32_t gpu_req; 21 uint32_t cpu_req; 22 uint32_t turn; 23 union { 24 struct { 25 uint16_t list_length; 26 uint16_t list_offset; 27 }; 28 struct { 29 uint8_t ifpc_list_len; 30 uint8_t preemption_list_len; 31 uint16_t dynamic_list_len; 32 }; 33 }; 34 uint64_t regs[62]; 35 }; 36 37 /** 38 * struct a6xx_info - a6xx specific information from device table 39 * 40 * @hwcg: hw clock gating register sequence 41 * @protect: CP_PROTECT settings 42 * @pwrup_reglist pwrup reglist for preemption 43 */ 44 struct a6xx_info { 45 const struct adreno_reglist *hwcg; 46 const struct adreno_protect *protect; 47 const struct adreno_reglist_list *pwrup_reglist; 48 u32 gmu_chipid; 49 u32 gmu_cgc_mode; 50 u32 prim_fifo_threshold; 51 const struct a6xx_bcm *bcms; 52 }; 53 54 struct a6xx_gpu { 55 struct adreno_gpu base; 56 57 struct drm_gem_object *sqe_bo; 58 uint64_t sqe_iova; 59 60 struct msm_ringbuffer *cur_ring; 61 struct msm_ringbuffer *next_ring; 62 63 struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS]; 64 void *preempt[MSM_GPU_MAX_RINGS]; 65 uint64_t preempt_iova[MSM_GPU_MAX_RINGS]; 66 struct drm_gem_object *preempt_smmu_bo[MSM_GPU_MAX_RINGS]; 67 void *preempt_smmu[MSM_GPU_MAX_RINGS]; 68 uint64_t preempt_smmu_iova[MSM_GPU_MAX_RINGS]; 69 uint32_t last_seqno[MSM_GPU_MAX_RINGS]; 70 71 atomic_t preempt_state; 72 spinlock_t eval_lock; 73 struct timer_list preempt_timer; 74 75 unsigned int preempt_level; 76 bool uses_gmem; 77 bool skip_save_restore; 78 79 struct drm_gem_object *preempt_postamble_bo; 80 void *preempt_postamble_ptr; 81 uint64_t preempt_postamble_iova; 82 uint64_t preempt_postamble_len; 83 bool postamble_enabled; 84 85 struct a6xx_gmu gmu; 86 87 struct drm_gem_object *shadow_bo; 88 uint64_t shadow_iova; 89 uint32_t *shadow; 90 91 struct drm_gem_object *pwrup_reglist_bo; 92 void *pwrup_reglist_ptr; 93 uint64_t pwrup_reglist_iova; 94 bool pwrup_reglist_emitted; 95 96 bool has_whereami; 97 98 void __iomem *llc_mmio; 99 void *llc_slice; 100 void *htw_llc_slice; 101 bool have_mmu500; 102 bool hung; 103 }; 104 105 #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base) 106 107 /* 108 * In order to do lockless preemption we use a simple state machine to progress 109 * through the process. 110 * 111 * PREEMPT_NONE - no preemption in progress. Next state START. 112 * PREEMPT_START - The trigger is evaluating if preemption is possible. Next 113 * states: TRIGGERED, NONE 114 * PREEMPT_FINISH - An intermediate state before moving back to NONE. Next 115 * state: NONE. 116 * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next 117 * states: FAULTED, PENDING 118 * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger 119 * recovery. Next state: N/A 120 * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is 121 * checking the success of the operation. Next state: FAULTED, NONE. 122 */ 123 124 enum a6xx_preempt_state { 125 PREEMPT_NONE = 0, 126 PREEMPT_START, 127 PREEMPT_FINISH, 128 PREEMPT_TRIGGERED, 129 PREEMPT_FAULTED, 130 PREEMPT_PENDING, 131 }; 132 133 /* 134 * struct a6xx_preempt_record is a shared buffer between the microcode and the 135 * CPU to store the state for preemption. The record itself is much larger 136 * (2112k) but most of that is used by the CP for storage. 137 * 138 * There is a preemption record assigned per ringbuffer. When the CPU triggers a 139 * preemption, it fills out the record with the useful information (wptr, ring 140 * base, etc) and the microcode uses that information to set up the CP following 141 * the preemption. When a ring is switched out, the CP will save the ringbuffer 142 * state back to the record. In this way, once the records are properly set up 143 * the CPU can quickly switch back and forth between ringbuffers by only 144 * updating a few registers (often only the wptr). 145 * 146 * These are the CPU aware registers in the record: 147 * @magic: Must always be 0xAE399D6EUL 148 * @info: Type of the record - written 0 by the CPU, updated by the CP 149 * @errno: preemption error record 150 * @data: Data field in YIELD and SET_MARKER packets, Written and used by CP 151 * @cntl: Value of RB_CNTL written by CPU, save/restored by CP 152 * @rptr: Value of RB_RPTR written by CPU, save/restored by CP 153 * @wptr: Value of RB_WPTR written by CPU, save/restored by CP 154 * @_pad: Reserved/padding 155 * @rptr_addr: Value of RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 156 * @rbase: Value of RB_BASE written by CPU, save/restored by CP 157 * @counter: GPU address of the storage area for the preemption counters 158 * @bv_rptr_addr: Value of BV_RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 159 */ 160 struct a6xx_preempt_record { 161 u32 magic; 162 u32 info; 163 u32 errno; 164 u32 data; 165 u32 cntl; 166 u32 rptr; 167 u32 wptr; 168 u32 _pad; 169 u64 rptr_addr; 170 u64 rbase; 171 u64 counter; 172 u64 bv_rptr_addr; 173 }; 174 175 #define A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6EUL 176 177 #define PREEMPT_SMMU_INFO_SIZE 4096 178 179 #define PREEMPT_RECORD_SIZE(adreno_gpu) \ 180 ((adreno_gpu->info->preempt_record_size) == 0 ? \ 181 4192 * SZ_1K : (adreno_gpu->info->preempt_record_size)) 182 183 /* 184 * The preemption counter block is a storage area for the value of the 185 * preemption counters that are saved immediately before context switch. We 186 * append it on to the end of the allocation for the preemption record. 187 */ 188 #define A6XX_PREEMPT_COUNTER_SIZE (16 * 4) 189 190 struct a7xx_cp_smmu_info { 191 u32 magic; 192 u32 _pad4; 193 u64 ttbr0; 194 u32 asid; 195 u32 context_idr; 196 u32 context_bank; 197 }; 198 199 #define GEN7_CP_SMMU_INFO_MAGIC 0x241350d5UL 200 201 /* 202 * Given a register and a count, return a value to program into 203 * REG_CP_PROTECT_REG(n) - this will block both reads and writes for 204 * _len + 1 registers starting at _reg. 205 */ 206 #define A6XX_PROTECT_NORDWR(_reg, _len) \ 207 ((1 << 31) | \ 208 (((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) 209 210 /* 211 * Same as above, but allow reads over the range. For areas of mixed use (such 212 * as performance counters) this allows us to protect a much larger range with a 213 * single register 214 */ 215 #define A6XX_PROTECT_RDONLY(_reg, _len) \ 216 ((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) 217 218 static inline bool a6xx_has_gbif(struct adreno_gpu *gpu) 219 { 220 if(adreno_is_a630(gpu)) 221 return false; 222 223 return true; 224 } 225 226 static inline void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or) 227 { 228 return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or); 229 } 230 231 static inline u32 a6xx_llc_read(struct a6xx_gpu *a6xx_gpu, u32 reg) 232 { 233 return readl(a6xx_gpu->llc_mmio + (reg << 2)); 234 } 235 236 static inline void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value) 237 { 238 writel(value, a6xx_gpu->llc_mmio + (reg << 2)); 239 } 240 241 #define shadowptr(_a6xx_gpu, _ring) ((_a6xx_gpu)->shadow_iova + \ 242 ((_ring)->id * sizeof(uint32_t))) 243 244 int a6xx_gmu_resume(struct a6xx_gpu *gpu); 245 int a6xx_gmu_stop(struct a6xx_gpu *gpu); 246 247 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu); 248 249 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu); 250 251 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); 252 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); 253 254 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 255 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 256 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu); 257 258 void a6xx_preempt_init(struct msm_gpu *gpu); 259 void a6xx_preempt_hw_init(struct msm_gpu *gpu); 260 void a6xx_preempt_trigger(struct msm_gpu *gpu); 261 void a6xx_preempt_irq(struct msm_gpu *gpu); 262 void a6xx_preempt_fini(struct msm_gpu *gpu); 263 int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu, 264 struct msm_gpu_submitqueue *queue); 265 void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu, 266 struct msm_gpu_submitqueue *queue); 267 268 /* Return true if we are in a preempt state */ 269 static inline bool a6xx_in_preempt(struct a6xx_gpu *a6xx_gpu) 270 { 271 /* 272 * Make sure the read to preempt_state is ordered with respect to reads 273 * of other variables before ... 274 */ 275 smp_rmb(); 276 277 int preempt_state = atomic_read(&a6xx_gpu->preempt_state); 278 279 /* ... and after. */ 280 smp_rmb(); 281 282 return !(preempt_state == PREEMPT_NONE || 283 preempt_state == PREEMPT_FINISH); 284 } 285 286 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, 287 bool suspended); 288 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu); 289 290 void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 291 struct drm_printer *p); 292 293 struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu); 294 int a6xx_gpu_state_put(struct msm_gpu_state *state); 295 296 void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off); 297 void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert); 298 299 #endif /* __A6XX_GPU_H__ */ 300