1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ 3 4 #include "msm_gem.h" 5 #include "a6xx_gpu.h" 6 #include "a6xx_gmu.xml.h" 7 #include "a6xx_preempt.h" 8 #include "msm_mmu.h" 9 #include "msm_gpu_trace.h" 10 11 static void preempt_prepare_postamble(struct a6xx_gpu *a6xx_gpu) 12 { 13 u32 *postamble = a6xx_gpu->preempt_postamble_ptr; 14 u32 count = 0; 15 16 postamble[count++] = PKT7(CP_REG_RMW, 3); 17 postamble[count++] = REG_A8XX_RBBM_PERFCTR_SRAM_INIT_CMD; 18 postamble[count++] = 0; 19 postamble[count++] = 1; 20 21 postamble[count++] = PKT7(CP_WAIT_REG_MEM, 6); 22 postamble[count++] = CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ); 23 postamble[count++] = CP_WAIT_REG_MEM_POLL_ADDR_LO( 24 REG_A8XX_RBBM_PERFCTR_SRAM_INIT_STATUS); 25 postamble[count++] = CP_WAIT_REG_MEM_POLL_ADDR_HI(0); 26 postamble[count++] = CP_WAIT_REG_MEM_3_REF(0x1); 27 postamble[count++] = CP_WAIT_REG_MEM_4_MASK(0x1); 28 postamble[count++] = CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0); 29 30 a6xx_gpu->preempt_postamble_len = count; 31 32 a6xx_gpu->postamble_enabled = true; 33 } 34 35 static void preempt_disable_postamble(struct a6xx_gpu *a6xx_gpu) 36 { 37 u32 *postamble = a6xx_gpu->preempt_postamble_ptr; 38 39 /* 40 * Disable the postamble by replacing the first packet header with a NOP 41 * that covers the whole buffer. 42 */ 43 *postamble = PKT7(CP_NOP, (a6xx_gpu->preempt_postamble_len - 1)); 44 45 a6xx_gpu->postamble_enabled = false; 46 } 47 48 /* 49 * Set preemption keepalive vote. Please note that this vote is different from the one used in 50 * a8xx_irq() 51 */ 52 static void a8xx_preempt_keepalive_vote(struct msm_gpu *gpu, bool on) 53 { 54 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 55 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 56 57 gmu_write(&a6xx_gpu->gmu, REG_A8XX_GMU_PWR_COL_PREEMPT_KEEPALIVE, on); 58 } 59 60 void a8xx_preempt_irq(struct msm_gpu *gpu) 61 { 62 uint32_t status; 63 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 64 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 65 struct drm_device *dev = gpu->dev; 66 67 if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 68 return; 69 70 /* Delete the preemption watchdog timer */ 71 timer_delete(&a6xx_gpu->preempt_timer); 72 73 /* 74 * The hardware should be setting the stop bit of CP_CONTEXT_SWITCH_CNTL 75 * to zero before firing the interrupt, but there is a non zero chance 76 * of a hardware condition or a software race that could set it again 77 * before we have a chance to finish. If that happens, log and go for 78 * recovery 79 */ 80 status = gpu_read(gpu, REG_A8XX_CP_CONTEXT_SWITCH_CNTL); 81 if (unlikely(status & A8XX_CP_CONTEXT_SWITCH_CNTL_STOP)) { 82 DRM_DEV_ERROR(&gpu->pdev->dev, 83 "!!!!!!!!!!!!!!!! preemption faulted !!!!!!!!!!!!!! irq\n"); 84 set_preempt_state(a6xx_gpu, PREEMPT_FAULTED); 85 dev_err(dev->dev, "%s: Preemption failed to complete\n", 86 gpu->name); 87 kthread_queue_work(gpu->worker, &gpu->recover_work); 88 return; 89 } 90 91 a6xx_gpu->cur_ring = a6xx_gpu->next_ring; 92 a6xx_gpu->next_ring = NULL; 93 94 set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 95 96 update_wptr(a6xx_gpu, a6xx_gpu->cur_ring); 97 98 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 99 100 a8xx_preempt_keepalive_vote(gpu, false); 101 102 trace_msm_gpu_preemption_irq(a6xx_gpu->cur_ring->id); 103 104 /* 105 * Retrigger preemption to avoid a deadlock that might occur when preemption 106 * is skipped due to it being already in flight when requested. 107 */ 108 a8xx_preempt_trigger(gpu); 109 } 110 111 void a8xx_preempt_hw_init(struct msm_gpu *gpu) 112 { 113 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 114 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 115 int i; 116 117 /* No preemption if we only have one ring */ 118 if (gpu->nr_rings == 1) 119 return; 120 121 for (i = 0; i < gpu->nr_rings; i++) { 122 struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[i]; 123 124 record_ptr->wptr = 0; 125 record_ptr->rptr = 0; 126 record_ptr->rptr_addr = shadowptr(a6xx_gpu, gpu->rb[i]); 127 record_ptr->info = 0; 128 record_ptr->data = 0; 129 record_ptr->rbase = gpu->rb[i]->iova; 130 } 131 132 /* Write a 0 to signal that we aren't switching pagetables */ 133 gpu_write64(gpu, REG_A8XX_CP_CONTEXT_SWITCH_SMMU_INFO, 0); 134 135 /* Enable the GMEM save/restore feature for preemption */ 136 gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE, 0x1); 137 138 /* Reset the preemption state */ 139 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 140 141 spin_lock_init(&a6xx_gpu->eval_lock); 142 143 /* Always come up on rb 0 */ 144 a6xx_gpu->cur_ring = gpu->rb[0]; 145 } 146 147 void a8xx_preempt_trigger(struct msm_gpu *gpu) 148 { 149 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 150 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 151 unsigned long flags; 152 struct msm_ringbuffer *ring; 153 unsigned int cntl; 154 bool sysprof; 155 156 if (gpu->nr_rings == 1) 157 return; 158 159 /* 160 * Lock to make sure another thread attempting preemption doesn't skip it 161 * while we are still evaluating the next ring. This makes sure the other 162 * thread does start preemption if we abort it and avoids a soft lock. 163 */ 164 spin_lock_irqsave(&a6xx_gpu->eval_lock, flags); 165 166 /* 167 * Try to start preemption by moving from NONE to START. If 168 * unsuccessful, a preemption is already in flight 169 */ 170 if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START)) { 171 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 172 return; 173 } 174 175 cntl = A8XX_CP_CONTEXT_SWITCH_CNTL_LEVEL(a6xx_gpu->preempt_level); 176 177 if (a6xx_gpu->skip_save_restore) 178 cntl |= A8XX_CP_CONTEXT_SWITCH_CNTL_SKIP_SAVE_RESTORE; 179 180 if (a6xx_gpu->uses_gmem) 181 cntl |= A8XX_CP_CONTEXT_SWITCH_CNTL_USES_GMEM; 182 183 cntl |= A8XX_CP_CONTEXT_SWITCH_CNTL_STOP; 184 185 /* Get the next ring to preempt to */ 186 ring = get_next_ring(gpu); 187 188 /* 189 * If no ring is populated or the highest priority ring is the current 190 * one do nothing except to update the wptr to the latest and greatest 191 */ 192 if (!ring || (a6xx_gpu->cur_ring == ring)) { 193 set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 194 update_wptr(a6xx_gpu, a6xx_gpu->cur_ring); 195 set_preempt_state(a6xx_gpu, PREEMPT_NONE); 196 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 197 return; 198 } 199 200 spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 201 202 spin_lock_irqsave(&ring->preempt_lock, flags); 203 204 struct a7xx_cp_smmu_info *smmu_info_ptr = 205 a6xx_gpu->preempt_smmu[ring->id]; 206 struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[ring->id]; 207 u64 ttbr0 = ring->memptrs->ttbr0; 208 u32 context_idr = ring->memptrs->context_idr; 209 210 smmu_info_ptr->ttbr0 = ttbr0; 211 smmu_info_ptr->context_idr = context_idr; 212 record_ptr->wptr = get_wptr(ring); 213 214 /* 215 * The GPU will write the wptr we set above when we preempt. Reset 216 * restore_wptr to make sure that we don't write WPTR to the same 217 * thing twice. It's still possible subsequent submissions will update 218 * wptr again, in which case they will set the flag to true. This has 219 * to be protected by the lock for setting the flag and updating wptr 220 * to be atomic. 221 */ 222 ring->restore_wptr = false; 223 224 trace_msm_gpu_preemption_trigger(a6xx_gpu->cur_ring->id, ring->id); 225 226 spin_unlock_irqrestore(&ring->preempt_lock, flags); 227 228 /* Set the keepalive bit to keep the GPU ON until preemption is complete */ 229 a8xx_preempt_keepalive_vote(gpu, true); 230 231 a6xx_fenced_write(a6xx_gpu, 232 REG_A8XX_CP_CONTEXT_SWITCH_SMMU_INFO, a6xx_gpu->preempt_smmu_iova[ring->id], 233 BIT(1), true); 234 235 a6xx_fenced_write(a6xx_gpu, 236 REG_A8XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR, 237 a6xx_gpu->preempt_iova[ring->id], BIT(1), true); 238 239 a6xx_gpu->next_ring = ring; 240 241 /* Start a timer to catch a stuck preemption */ 242 mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 243 244 /* Enable or disable postamble as needed */ 245 sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1; 246 247 if (!sysprof && !a6xx_gpu->postamble_enabled) 248 preempt_prepare_postamble(a6xx_gpu); 249 250 if (sysprof && a6xx_gpu->postamble_enabled) 251 preempt_disable_postamble(a6xx_gpu); 252 253 /* Set the preemption state to triggered */ 254 set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED); 255 256 /* Trigger the preemption */ 257 a6xx_fenced_write(a6xx_gpu, REG_A8XX_CP_CONTEXT_SWITCH_CNTL, cntl, BIT(1), false); 258 } 259 260