1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2017, 2019 The Linux Foundation. All rights reserved. */ 3 4 #ifndef __A6XX_GPU_H__ 5 #define __A6XX_GPU_H__ 6 7 8 #include "adreno_gpu.h" 9 #include "a6xx_enums.xml.h" 10 #include "a7xx_enums.xml.h" 11 #include "a6xx_perfcntrs.xml.h" 12 #include "a7xx_perfcntrs.xml.h" 13 #include "a6xx.xml.h" 14 15 #include "a6xx_gmu.h" 16 17 extern bool hang_debug; 18 19 struct cpu_gpu_lock { 20 uint32_t gpu_req; 21 uint32_t cpu_req; 22 uint32_t turn; 23 union { 24 struct { 25 uint16_t list_length; 26 uint16_t list_offset; 27 }; 28 struct { 29 uint8_t ifpc_list_len; 30 uint8_t preemption_list_len; 31 uint16_t dynamic_list_len; 32 }; 33 }; 34 uint64_t regs[62]; 35 }; 36 37 /** 38 * struct a6xx_info - a6xx specific information from device table 39 * 40 * @hwcg: hw clock gating register sequence 41 * @protect: CP_PROTECT settings 42 * @pwrup_reglist pwrup reglist for preemption 43 */ 44 struct a6xx_info { 45 const struct adreno_reglist *hwcg; 46 const struct adreno_protect *protect; 47 const struct adreno_reglist_list *pwrup_reglist; 48 const struct adreno_reglist_list *ifpc_reglist; 49 u32 gmu_chipid; 50 u32 gmu_cgc_mode; 51 u32 prim_fifo_threshold; 52 const struct a6xx_bcm *bcms; 53 }; 54 55 struct a6xx_gpu { 56 struct adreno_gpu base; 57 58 struct drm_gem_object *sqe_bo; 59 uint64_t sqe_iova; 60 61 struct msm_ringbuffer *cur_ring; 62 struct msm_ringbuffer *next_ring; 63 64 struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS]; 65 void *preempt[MSM_GPU_MAX_RINGS]; 66 uint64_t preempt_iova[MSM_GPU_MAX_RINGS]; 67 struct drm_gem_object *preempt_smmu_bo[MSM_GPU_MAX_RINGS]; 68 void *preempt_smmu[MSM_GPU_MAX_RINGS]; 69 uint64_t preempt_smmu_iova[MSM_GPU_MAX_RINGS]; 70 uint32_t last_seqno[MSM_GPU_MAX_RINGS]; 71 72 atomic_t preempt_state; 73 spinlock_t eval_lock; 74 struct timer_list preempt_timer; 75 76 unsigned int preempt_level; 77 bool uses_gmem; 78 bool skip_save_restore; 79 80 struct drm_gem_object *preempt_postamble_bo; 81 void *preempt_postamble_ptr; 82 uint64_t preempt_postamble_iova; 83 uint64_t preempt_postamble_len; 84 bool postamble_enabled; 85 86 struct a6xx_gmu gmu; 87 88 struct drm_gem_object *shadow_bo; 89 uint64_t shadow_iova; 90 uint32_t *shadow; 91 92 struct drm_gem_object *pwrup_reglist_bo; 93 void *pwrup_reglist_ptr; 94 uint64_t pwrup_reglist_iova; 95 bool pwrup_reglist_emitted; 96 97 bool has_whereami; 98 99 void __iomem *llc_mmio; 100 void *llc_slice; 101 void *htw_llc_slice; 102 bool have_mmu500; 103 bool hung; 104 }; 105 106 #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base) 107 108 /* 109 * In order to do lockless preemption we use a simple state machine to progress 110 * through the process. 111 * 112 * PREEMPT_NONE - no preemption in progress. Next state START. 113 * PREEMPT_START - The trigger is evaluating if preemption is possible. Next 114 * states: TRIGGERED, NONE 115 * PREEMPT_FINISH - An intermediate state before moving back to NONE. Next 116 * state: NONE. 117 * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next 118 * states: FAULTED, PENDING 119 * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger 120 * recovery. Next state: N/A 121 * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is 122 * checking the success of the operation. Next state: FAULTED, NONE. 123 */ 124 125 enum a6xx_preempt_state { 126 PREEMPT_NONE = 0, 127 PREEMPT_START, 128 PREEMPT_FINISH, 129 PREEMPT_TRIGGERED, 130 PREEMPT_FAULTED, 131 PREEMPT_PENDING, 132 }; 133 134 /* 135 * struct a6xx_preempt_record is a shared buffer between the microcode and the 136 * CPU to store the state for preemption. The record itself is much larger 137 * (2112k) but most of that is used by the CP for storage. 138 * 139 * There is a preemption record assigned per ringbuffer. When the CPU triggers a 140 * preemption, it fills out the record with the useful information (wptr, ring 141 * base, etc) and the microcode uses that information to set up the CP following 142 * the preemption. When a ring is switched out, the CP will save the ringbuffer 143 * state back to the record. In this way, once the records are properly set up 144 * the CPU can quickly switch back and forth between ringbuffers by only 145 * updating a few registers (often only the wptr). 146 * 147 * These are the CPU aware registers in the record: 148 * @magic: Must always be 0xAE399D6EUL 149 * @info: Type of the record - written 0 by the CPU, updated by the CP 150 * @errno: preemption error record 151 * @data: Data field in YIELD and SET_MARKER packets, Written and used by CP 152 * @cntl: Value of RB_CNTL written by CPU, save/restored by CP 153 * @rptr: Value of RB_RPTR written by CPU, save/restored by CP 154 * @wptr: Value of RB_WPTR written by CPU, save/restored by CP 155 * @_pad: Reserved/padding 156 * @rptr_addr: Value of RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 157 * @rbase: Value of RB_BASE written by CPU, save/restored by CP 158 * @counter: GPU address of the storage area for the preemption counters 159 * @bv_rptr_addr: Value of BV_RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 160 */ 161 struct a6xx_preempt_record { 162 u32 magic; 163 u32 info; 164 u32 errno; 165 u32 data; 166 u32 cntl; 167 u32 rptr; 168 u32 wptr; 169 u32 _pad; 170 u64 rptr_addr; 171 u64 rbase; 172 u64 counter; 173 u64 bv_rptr_addr; 174 }; 175 176 #define A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6EUL 177 178 #define PREEMPT_SMMU_INFO_SIZE 4096 179 180 #define PREEMPT_RECORD_SIZE(adreno_gpu) \ 181 ((adreno_gpu->info->preempt_record_size) == 0 ? \ 182 4192 * SZ_1K : (adreno_gpu->info->preempt_record_size)) 183 184 /* 185 * The preemption counter block is a storage area for the value of the 186 * preemption counters that are saved immediately before context switch. We 187 * append it on to the end of the allocation for the preemption record. 188 */ 189 #define A6XX_PREEMPT_COUNTER_SIZE (16 * 4) 190 191 struct a7xx_cp_smmu_info { 192 u32 magic; 193 u32 _pad4; 194 u64 ttbr0; 195 u32 asid; 196 u32 context_idr; 197 u32 context_bank; 198 }; 199 200 #define GEN7_CP_SMMU_INFO_MAGIC 0x241350d5UL 201 202 /* 203 * Given a register and a count, return a value to program into 204 * REG_CP_PROTECT_REG(n) - this will block both reads and writes for 205 * _len + 1 registers starting at _reg. 206 */ 207 #define A6XX_PROTECT_NORDWR(_reg, _len) \ 208 ((1 << 31) | \ 209 (((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) 210 211 /* 212 * Same as above, but allow reads over the range. For areas of mixed use (such 213 * as performance counters) this allows us to protect a much larger range with a 214 * single register 215 */ 216 #define A6XX_PROTECT_RDONLY(_reg, _len) \ 217 ((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) 218 219 static inline bool a6xx_has_gbif(struct adreno_gpu *gpu) 220 { 221 if(adreno_is_a630(gpu)) 222 return false; 223 224 return true; 225 } 226 227 static inline void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or) 228 { 229 return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or); 230 } 231 232 static inline u32 a6xx_llc_read(struct a6xx_gpu *a6xx_gpu, u32 reg) 233 { 234 return readl(a6xx_gpu->llc_mmio + (reg << 2)); 235 } 236 237 static inline void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value) 238 { 239 writel(value, a6xx_gpu->llc_mmio + (reg << 2)); 240 } 241 242 #define shadowptr(_a6xx_gpu, _ring) ((_a6xx_gpu)->shadow_iova + \ 243 ((_ring)->id * sizeof(uint32_t))) 244 245 int a6xx_gmu_resume(struct a6xx_gpu *gpu); 246 int a6xx_gmu_stop(struct a6xx_gpu *gpu); 247 248 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu); 249 250 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu); 251 252 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); 253 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); 254 255 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 256 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 257 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu); 258 void a6xx_gmu_sysprof_setup(struct msm_gpu *gpu); 259 260 void a6xx_preempt_init(struct msm_gpu *gpu); 261 void a6xx_preempt_hw_init(struct msm_gpu *gpu); 262 void a6xx_preempt_trigger(struct msm_gpu *gpu); 263 void a6xx_preempt_irq(struct msm_gpu *gpu); 264 void a6xx_preempt_fini(struct msm_gpu *gpu); 265 int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu, 266 struct msm_gpu_submitqueue *queue); 267 void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu, 268 struct msm_gpu_submitqueue *queue); 269 270 /* Return true if we are in a preempt state */ 271 static inline bool a6xx_in_preempt(struct a6xx_gpu *a6xx_gpu) 272 { 273 /* 274 * Make sure the read to preempt_state is ordered with respect to reads 275 * of other variables before ... 276 */ 277 smp_rmb(); 278 279 int preempt_state = atomic_read(&a6xx_gpu->preempt_state); 280 281 /* ... and after. */ 282 smp_rmb(); 283 284 return !(preempt_state == PREEMPT_NONE || 285 preempt_state == PREEMPT_FINISH); 286 } 287 288 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, 289 bool suspended); 290 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu); 291 292 void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, 293 struct drm_printer *p); 294 295 struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu); 296 int a6xx_gpu_state_put(struct msm_gpu_state *state); 297 298 void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off); 299 void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert); 300 int a6xx_fenced_write(struct a6xx_gpu *gpu, u32 offset, u64 value, u32 mask, bool is_64b); 301 302 #endif /* __A6XX_GPU_H__ */ 303