1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2017, 2019 The Linux Foundation. All rights reserved. */
3
4 #ifndef __A6XX_GPU_H__
5 #define __A6XX_GPU_H__
6
7
8 #include "adreno_gpu.h"
9 #include "a6xx.xml.h"
10
11 #include "a6xx_gmu.h"
12
13 extern bool hang_debug;
14
15 struct cpu_gpu_lock {
16 uint32_t gpu_req;
17 uint32_t cpu_req;
18 uint32_t turn;
19 union {
20 struct {
21 uint16_t list_length;
22 uint16_t list_offset;
23 };
24 struct {
25 uint8_t ifpc_list_len;
26 uint8_t preemption_list_len;
27 uint16_t dynamic_list_len;
28 };
29 };
30 uint64_t regs[62];
31 };
32
33 /**
34 * struct a6xx_info - a6xx specific information from device table
35 *
36 * @hwcg: hw clock gating register sequence
37 * @protect: CP_PROTECT settings
38 * @pwrup_reglist pwrup reglist for preemption
39 */
40 struct a6xx_info {
41 const struct adreno_reglist *hwcg;
42 const struct adreno_protect *protect;
43 const struct adreno_reglist_list *pwrup_reglist;
44 u32 gmu_chipid;
45 u32 gmu_cgc_mode;
46 u32 prim_fifo_threshold;
47 const struct a6xx_bcm *bcms;
48 };
49
50 struct a6xx_gpu {
51 struct adreno_gpu base;
52
53 struct drm_gem_object *sqe_bo;
54 uint64_t sqe_iova;
55
56 struct msm_ringbuffer *cur_ring;
57 struct msm_ringbuffer *next_ring;
58
59 struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS];
60 void *preempt[MSM_GPU_MAX_RINGS];
61 uint64_t preempt_iova[MSM_GPU_MAX_RINGS];
62 struct drm_gem_object *preempt_smmu_bo[MSM_GPU_MAX_RINGS];
63 void *preempt_smmu[MSM_GPU_MAX_RINGS];
64 uint64_t preempt_smmu_iova[MSM_GPU_MAX_RINGS];
65 uint32_t last_seqno[MSM_GPU_MAX_RINGS];
66
67 atomic_t preempt_state;
68 spinlock_t eval_lock;
69 struct timer_list preempt_timer;
70
71 unsigned int preempt_level;
72 bool uses_gmem;
73 bool skip_save_restore;
74
75 struct drm_gem_object *preempt_postamble_bo;
76 void *preempt_postamble_ptr;
77 uint64_t preempt_postamble_iova;
78 uint64_t preempt_postamble_len;
79 bool postamble_enabled;
80
81 struct a6xx_gmu gmu;
82
83 struct drm_gem_object *shadow_bo;
84 uint64_t shadow_iova;
85 uint32_t *shadow;
86
87 struct drm_gem_object *pwrup_reglist_bo;
88 void *pwrup_reglist_ptr;
89 uint64_t pwrup_reglist_iova;
90 bool pwrup_reglist_emitted;
91
92 bool has_whereami;
93
94 void __iomem *llc_mmio;
95 void *llc_slice;
96 void *htw_llc_slice;
97 bool have_mmu500;
98 bool hung;
99 };
100
101 #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base)
102
103 /*
104 * In order to do lockless preemption we use a simple state machine to progress
105 * through the process.
106 *
107 * PREEMPT_NONE - no preemption in progress. Next state START.
108 * PREEMPT_START - The trigger is evaluating if preemption is possible. Next
109 * states: TRIGGERED, NONE
110 * PREEMPT_FINISH - An intermediate state before moving back to NONE. Next
111 * state: NONE.
112 * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next
113 * states: FAULTED, PENDING
114 * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger
115 * recovery. Next state: N/A
116 * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is
117 * checking the success of the operation. Next state: FAULTED, NONE.
118 */
119
120 enum a6xx_preempt_state {
121 PREEMPT_NONE = 0,
122 PREEMPT_START,
123 PREEMPT_FINISH,
124 PREEMPT_TRIGGERED,
125 PREEMPT_FAULTED,
126 PREEMPT_PENDING,
127 };
128
129 /*
130 * struct a6xx_preempt_record is a shared buffer between the microcode and the
131 * CPU to store the state for preemption. The record itself is much larger
132 * (2112k) but most of that is used by the CP for storage.
133 *
134 * There is a preemption record assigned per ringbuffer. When the CPU triggers a
135 * preemption, it fills out the record with the useful information (wptr, ring
136 * base, etc) and the microcode uses that information to set up the CP following
137 * the preemption. When a ring is switched out, the CP will save the ringbuffer
138 * state back to the record. In this way, once the records are properly set up
139 * the CPU can quickly switch back and forth between ringbuffers by only
140 * updating a few registers (often only the wptr).
141 *
142 * These are the CPU aware registers in the record:
143 * @magic: Must always be 0xAE399D6EUL
144 * @info: Type of the record - written 0 by the CPU, updated by the CP
145 * @errno: preemption error record
146 * @data: Data field in YIELD and SET_MARKER packets, Written and used by CP
147 * @cntl: Value of RB_CNTL written by CPU, save/restored by CP
148 * @rptr: Value of RB_RPTR written by CPU, save/restored by CP
149 * @wptr: Value of RB_WPTR written by CPU, save/restored by CP
150 * @_pad: Reserved/padding
151 * @rptr_addr: Value of RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP
152 * @rbase: Value of RB_BASE written by CPU, save/restored by CP
153 * @counter: GPU address of the storage area for the preemption counters
154 * @bv_rptr_addr: Value of BV_RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP
155 */
156 struct a6xx_preempt_record {
157 u32 magic;
158 u32 info;
159 u32 errno;
160 u32 data;
161 u32 cntl;
162 u32 rptr;
163 u32 wptr;
164 u32 _pad;
165 u64 rptr_addr;
166 u64 rbase;
167 u64 counter;
168 u64 bv_rptr_addr;
169 };
170
171 #define A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6EUL
172
173 #define PREEMPT_SMMU_INFO_SIZE 4096
174
175 #define PREEMPT_RECORD_SIZE(adreno_gpu) \
176 ((adreno_gpu->info->preempt_record_size) == 0 ? \
177 4192 * SZ_1K : (adreno_gpu->info->preempt_record_size))
178
179 /*
180 * The preemption counter block is a storage area for the value of the
181 * preemption counters that are saved immediately before context switch. We
182 * append it on to the end of the allocation for the preemption record.
183 */
184 #define A6XX_PREEMPT_COUNTER_SIZE (16 * 4)
185
186 struct a7xx_cp_smmu_info {
187 u32 magic;
188 u32 _pad4;
189 u64 ttbr0;
190 u32 asid;
191 u32 context_idr;
192 u32 context_bank;
193 };
194
195 #define GEN7_CP_SMMU_INFO_MAGIC 0x241350d5UL
196
197 /*
198 * Given a register and a count, return a value to program into
199 * REG_CP_PROTECT_REG(n) - this will block both reads and writes for
200 * _len + 1 registers starting at _reg.
201 */
202 #define A6XX_PROTECT_NORDWR(_reg, _len) \
203 ((1 << 31) | \
204 (((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF))
205
206 /*
207 * Same as above, but allow reads over the range. For areas of mixed use (such
208 * as performance counters) this allows us to protect a much larger range with a
209 * single register
210 */
211 #define A6XX_PROTECT_RDONLY(_reg, _len) \
212 ((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF))
213
a6xx_has_gbif(struct adreno_gpu * gpu)214 static inline bool a6xx_has_gbif(struct adreno_gpu *gpu)
215 {
216 if(adreno_is_a630(gpu))
217 return false;
218
219 return true;
220 }
221
a6xx_llc_rmw(struct a6xx_gpu * a6xx_gpu,u32 reg,u32 mask,u32 or)222 static inline void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or)
223 {
224 return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or);
225 }
226
a6xx_llc_read(struct a6xx_gpu * a6xx_gpu,u32 reg)227 static inline u32 a6xx_llc_read(struct a6xx_gpu *a6xx_gpu, u32 reg)
228 {
229 return readl(a6xx_gpu->llc_mmio + (reg << 2));
230 }
231
a6xx_llc_write(struct a6xx_gpu * a6xx_gpu,u32 reg,u32 value)232 static inline void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value)
233 {
234 writel(value, a6xx_gpu->llc_mmio + (reg << 2));
235 }
236
237 #define shadowptr(_a6xx_gpu, _ring) ((_a6xx_gpu)->shadow_iova + \
238 ((_ring)->id * sizeof(uint32_t)))
239
240 int a6xx_gmu_resume(struct a6xx_gpu *gpu);
241 int a6xx_gmu_stop(struct a6xx_gpu *gpu);
242
243 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu);
244
245 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu);
246
247 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
248 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
249
250 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node);
251 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node);
252 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu);
253
254 void a6xx_preempt_init(struct msm_gpu *gpu);
255 void a6xx_preempt_hw_init(struct msm_gpu *gpu);
256 void a6xx_preempt_trigger(struct msm_gpu *gpu);
257 void a6xx_preempt_irq(struct msm_gpu *gpu);
258 void a6xx_preempt_fini(struct msm_gpu *gpu);
259 int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu,
260 struct msm_gpu_submitqueue *queue);
261 void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu,
262 struct msm_gpu_submitqueue *queue);
263
264 /* Return true if we are in a preempt state */
a6xx_in_preempt(struct a6xx_gpu * a6xx_gpu)265 static inline bool a6xx_in_preempt(struct a6xx_gpu *a6xx_gpu)
266 {
267 /*
268 * Make sure the read to preempt_state is ordered with respect to reads
269 * of other variables before ...
270 */
271 smp_rmb();
272
273 int preempt_state = atomic_read(&a6xx_gpu->preempt_state);
274
275 /* ... and after. */
276 smp_rmb();
277
278 return !(preempt_state == PREEMPT_NONE ||
279 preempt_state == PREEMPT_FINISH);
280 }
281
282 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
283 bool suspended);
284 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu);
285
286 void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
287 struct drm_printer *p);
288
289 struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu);
290 int a6xx_gpu_state_put(struct msm_gpu_state *state);
291
292 void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off);
293 void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert);
294
295 #endif /* __A6XX_GPU_H__ */
296