1 /* 2 * Copyright 2015 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 25 #include "amdgpu.h" 26 #include "amdgpu_trace.h" 27 #include "si.h" 28 #include "sid.h" 29 30 #include "oss/oss_1_0_d.h" 31 #include "oss/oss_1_0_sh_mask.h" 32 const u32 sdma_offsets[SDMA_MAX_INSTANCE] = 33 { 34 DMA0_REGISTER_OFFSET, 35 DMA1_REGISTER_OFFSET 36 }; 37 38 static void si_dma_set_ring_funcs(struct amdgpu_device *adev); 39 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev); 40 static void si_dma_set_irq_funcs(struct amdgpu_device *adev); 41 42 /** 43 * si_dma_ring_get_rptr - get the current read pointer 44 * 45 * @ring: amdgpu ring pointer 46 * 47 * Get the current rptr from the hardware (SI). 48 */ 49 static uint64_t si_dma_ring_get_rptr(struct amdgpu_ring *ring) 50 { 51 return *ring->rptr_cpu_addr; 52 } 53 54 /** 55 * si_dma_ring_get_wptr - get the current write pointer 56 * 57 * @ring: amdgpu ring pointer 58 * 59 * Get the current wptr from the hardware (SI). 60 */ 61 static uint64_t si_dma_ring_get_wptr(struct amdgpu_ring *ring) 62 { 63 struct amdgpu_device *adev = ring->adev; 64 u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1; 65 66 return (RREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[me]) & 0x3fffc) >> 2; 67 } 68 69 static void si_dma_ring_set_wptr(struct amdgpu_ring *ring) 70 { 71 struct amdgpu_device *adev = ring->adev; 72 u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1; 73 74 WREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc); 75 } 76 77 static void si_dma_ring_emit_ib(struct amdgpu_ring *ring, 78 struct amdgpu_job *job, 79 struct amdgpu_ib *ib, 80 uint32_t flags) 81 { 82 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 83 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. 84 * Pad as necessary with NOPs. 85 */ 86 while ((lower_32_bits(ring->wptr) & 7) != 5) 87 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 88 amdgpu_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vmid, 0)); 89 amdgpu_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); 90 amdgpu_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); 91 92 } 93 94 /** 95 * si_dma_ring_emit_fence - emit a fence on the DMA ring 96 * 97 * @ring: amdgpu ring pointer 98 * @addr: address 99 * @seq: sequence number 100 * @flags: fence related flags 101 * 102 * Add a DMA fence packet to the ring to write 103 * the fence seq number and DMA trap packet to generate 104 * an interrupt if needed (VI). 105 */ 106 static void si_dma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, 107 unsigned flags) 108 { 109 110 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 111 /* write the fence */ 112 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0)); 113 amdgpu_ring_write(ring, addr & 0xfffffffc); 114 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff)); 115 amdgpu_ring_write(ring, seq); 116 /* optionally write high bits as well */ 117 if (write64bit) { 118 addr += 4; 119 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0)); 120 amdgpu_ring_write(ring, addr & 0xfffffffc); 121 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff)); 122 amdgpu_ring_write(ring, upper_32_bits(seq)); 123 } 124 /* generate an interrupt */ 125 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0, 0)); 126 } 127 128 static void si_dma_stop(struct amdgpu_device *adev) 129 { 130 u32 rb_cntl; 131 unsigned i; 132 133 for (i = 0; i < adev->sdma.num_instances; i++) { 134 /* dma0 */ 135 rb_cntl = RREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i]); 136 rb_cntl &= ~DMA_GFX_RB_CNTL__RB_ENABLE_MASK; 137 WREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); 138 } 139 } 140 141 static int si_dma_start(struct amdgpu_device *adev) 142 { 143 struct amdgpu_ring *ring; 144 u32 rb_cntl, dma_cntl, ib_cntl, rb_bufsz; 145 int i, r; 146 uint64_t rptr_addr; 147 148 for (i = 0; i < adev->sdma.num_instances; i++) { 149 ring = &adev->sdma.instance[i].ring; 150 151 WREG32(mmDMA_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0); 152 WREG32(mmDMA_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0); 153 154 /* Set ring buffer size in dwords */ 155 rb_bufsz = order_base_2(ring->ring_size / 4); 156 rb_cntl = rb_bufsz << 1; 157 #ifdef __BIG_ENDIAN 158 rb_cntl |= DMA_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK | DMA_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK; 159 #endif 160 WREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); 161 162 /* Initialize the ring buffer's read and write pointers */ 163 WREG32(mmDMA_GFX_RB_RPTR + sdma_offsets[i], 0); 164 WREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[i], 0); 165 166 rptr_addr = ring->rptr_gpu_addr; 167 168 WREG32(mmDMA_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i], lower_32_bits(rptr_addr)); 169 WREG32(mmDMA_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i], upper_32_bits(rptr_addr) & 0xFF); 170 171 rb_cntl |= DMA_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK; 172 173 WREG32(mmDMA_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8); 174 175 /* enable DMA IBs */ 176 ib_cntl = DMA_GFX_IB_CNTL__IB_ENABLE_MASK | DMA_GFX_IB_CNTL__CMD_VMID_FORCE_MASK; 177 #ifdef __BIG_ENDIAN 178 ib_cntl |= DMA_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK; 179 #endif 180 WREG32(mmDMA_GFX_IB_CNTL + sdma_offsets[i], ib_cntl); 181 182 dma_cntl = RREG32(mmDMA_CNTL + sdma_offsets[i]); 183 dma_cntl &= ~DMA_CNTL__CTXEMPTY_INT_ENABLE_MASK; 184 WREG32(mmDMA_CNTL + sdma_offsets[i], dma_cntl); 185 186 ring->wptr = 0; 187 WREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[i], ring->wptr << 2); 188 WREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i], rb_cntl | DMA_GFX_RB_CNTL__RB_ENABLE_MASK); 189 190 r = amdgpu_ring_test_helper(ring); 191 if (r) 192 return r; 193 } 194 195 return 0; 196 } 197 198 /** 199 * si_dma_ring_test_ring - simple async dma engine test 200 * 201 * @ring: amdgpu_ring structure holding ring information 202 * 203 * Test the DMA engine by writing using it to write an 204 * value to memory. (VI). 205 * Returns 0 for success, error for failure. 206 */ 207 static int si_dma_ring_test_ring(struct amdgpu_ring *ring) 208 { 209 struct amdgpu_device *adev = ring->adev; 210 unsigned i; 211 unsigned index; 212 int r; 213 u32 tmp; 214 u64 gpu_addr; 215 216 r = amdgpu_wb_get(adev, &index); 217 if (r) 218 return r; 219 220 gpu_addr = adev->wb.gpu_addr + (index * 4); 221 tmp = 0xCAFEDEAD; 222 adev->wb.wb[index] = cpu_to_le32(tmp); 223 224 r = amdgpu_ring_alloc(ring, 4); 225 if (r) 226 goto error_free_wb; 227 228 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1)); 229 amdgpu_ring_write(ring, lower_32_bits(gpu_addr)); 230 amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xff); 231 amdgpu_ring_write(ring, 0xDEADBEEF); 232 amdgpu_ring_commit(ring); 233 234 for (i = 0; i < adev->usec_timeout; i++) { 235 tmp = le32_to_cpu(adev->wb.wb[index]); 236 if (tmp == 0xDEADBEEF) 237 break; 238 udelay(1); 239 } 240 241 if (i >= adev->usec_timeout) 242 r = -ETIMEDOUT; 243 244 error_free_wb: 245 amdgpu_wb_free(adev, index); 246 return r; 247 } 248 249 /** 250 * si_dma_ring_test_ib - test an IB on the DMA engine 251 * 252 * @ring: amdgpu_ring structure holding ring information 253 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT 254 * 255 * Test a simple IB in the DMA ring (VI). 256 * Returns 0 on success, error on failure. 257 */ 258 static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout) 259 { 260 struct amdgpu_device *adev = ring->adev; 261 struct amdgpu_job *job; 262 struct amdgpu_ib *ib; 263 struct dma_fence *f = NULL; 264 unsigned index; 265 u32 tmp = 0; 266 u64 gpu_addr; 267 long r; 268 269 r = amdgpu_wb_get(adev, &index); 270 if (r) 271 return r; 272 273 gpu_addr = adev->wb.gpu_addr + (index * 4); 274 tmp = 0xCAFEDEAD; 275 adev->wb.wb[index] = cpu_to_le32(tmp); 276 277 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256, 278 AMDGPU_IB_POOL_DIRECT, 279 AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST, 280 &job); 281 if (r) 282 goto err0; 283 284 ib = &job->ibs[0]; 285 ib->ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1); 286 ib->ptr[1] = lower_32_bits(gpu_addr); 287 ib->ptr[2] = upper_32_bits(gpu_addr) & 0xff; 288 ib->ptr[3] = 0xDEADBEEF; 289 ib->length_dw = 4; 290 291 r = amdgpu_job_submit_direct(job, ring, &f); 292 if (r) { 293 amdgpu_job_free(job); 294 goto err1; 295 } 296 297 r = dma_fence_wait_timeout(f, false, timeout); 298 if (r == 0) { 299 r = -ETIMEDOUT; 300 goto err1; 301 } else if (r < 0) { 302 goto err1; 303 } 304 tmp = le32_to_cpu(adev->wb.wb[index]); 305 if (tmp == 0xDEADBEEF) 306 r = 0; 307 else 308 r = -EINVAL; 309 310 err1: 311 dma_fence_put(f); 312 err0: 313 amdgpu_wb_free(adev, index); 314 return r; 315 } 316 317 /** 318 * si_dma_vm_copy_pte - update PTEs by copying them from the GART 319 * 320 * @ib: indirect buffer to fill with commands 321 * @pe: addr of the page entry 322 * @src: src addr to copy from 323 * @count: number of page entries to update 324 * 325 * Update PTEs by copying them from the GART using DMA (SI). 326 */ 327 static void si_dma_vm_copy_pte(struct amdgpu_ib *ib, 328 uint64_t pe, uint64_t src, 329 unsigned count) 330 { 331 unsigned bytes = count * 8; 332 333 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY, 334 1, 0, 0, bytes); 335 ib->ptr[ib->length_dw++] = lower_32_bits(pe); 336 ib->ptr[ib->length_dw++] = lower_32_bits(src); 337 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 338 ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff; 339 } 340 341 /** 342 * si_dma_vm_write_pte - update PTEs by writing them manually 343 * 344 * @ib: indirect buffer to fill with commands 345 * @pe: addr of the page entry 346 * @value: dst addr to write into pe 347 * @count: number of page entries to update 348 * @incr: increase next addr by incr bytes 349 * 350 * Update PTEs by writing them manually using DMA (SI). 351 */ 352 static void si_dma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, 353 uint64_t value, unsigned count, 354 uint32_t incr) 355 { 356 unsigned ndw = count * 2; 357 358 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw); 359 ib->ptr[ib->length_dw++] = lower_32_bits(pe); 360 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 361 for (; ndw > 0; ndw -= 2) { 362 ib->ptr[ib->length_dw++] = lower_32_bits(value); 363 ib->ptr[ib->length_dw++] = upper_32_bits(value); 364 value += incr; 365 } 366 } 367 368 /** 369 * si_dma_vm_set_pte_pde - update the page tables using sDMA 370 * 371 * @ib: indirect buffer to fill with commands 372 * @pe: addr of the page entry 373 * @addr: dst addr to write into pe 374 * @count: number of page entries to update 375 * @incr: increase next addr by incr bytes 376 * @flags: access flags 377 * 378 * Update the page tables using sDMA (CIK). 379 */ 380 static void si_dma_vm_set_pte_pde(struct amdgpu_ib *ib, 381 uint64_t pe, 382 uint64_t addr, unsigned count, 383 uint32_t incr, uint64_t flags) 384 { 385 uint64_t value; 386 unsigned ndw; 387 388 while (count) { 389 ndw = count * 2; 390 if (ndw > 0xFFFFE) 391 ndw = 0xFFFFE; 392 393 if (flags & AMDGPU_PTE_VALID) 394 value = addr; 395 else 396 value = 0; 397 398 /* for physically contiguous pages (vram) */ 399 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); 400 ib->ptr[ib->length_dw++] = pe; /* dst addr */ 401 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 402 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */ 403 ib->ptr[ib->length_dw++] = upper_32_bits(flags); 404 ib->ptr[ib->length_dw++] = value; /* value */ 405 ib->ptr[ib->length_dw++] = upper_32_bits(value); 406 ib->ptr[ib->length_dw++] = incr; /* increment size */ 407 ib->ptr[ib->length_dw++] = 0; 408 pe += ndw * 4; 409 addr += (ndw / 2) * incr; 410 count -= ndw / 2; 411 } 412 } 413 414 /** 415 * si_dma_ring_pad_ib - pad the IB to the required number of dw 416 * 417 * @ring: amdgpu_ring pointer 418 * @ib: indirect buffer to fill with padding 419 * 420 */ 421 static void si_dma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) 422 { 423 while (ib->length_dw & 0x7) 424 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0); 425 } 426 427 /** 428 * si_dma_ring_emit_pipeline_sync - sync the pipeline 429 * 430 * @ring: amdgpu_ring pointer 431 * 432 * Make sure all previous operations are completed (CIK). 433 */ 434 static void si_dma_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 435 { 436 uint32_t seq = ring->fence_drv.sync_seq; 437 uint64_t addr = ring->fence_drv.gpu_addr; 438 439 /* wait for idle */ 440 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0) | 441 (1 << 27)); /* Poll memory */ 442 amdgpu_ring_write(ring, lower_32_bits(addr)); 443 amdgpu_ring_write(ring, (0xff << 16) | upper_32_bits(addr)); /* retry, addr_hi */ 444 amdgpu_ring_write(ring, 0xffffffff); /* mask */ 445 amdgpu_ring_write(ring, seq); /* value */ 446 amdgpu_ring_write(ring, (3 << 28) | 0x20); /* func(equal) | poll interval */ 447 } 448 449 /** 450 * si_dma_ring_emit_vm_flush - cik vm flush using sDMA 451 * 452 * @ring: amdgpu_ring pointer 453 * @vmid: vmid number to use 454 * @pd_addr: address 455 * 456 * Update the page table base and flush the VM TLB 457 * using sDMA (VI). 458 */ 459 static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring, 460 unsigned vmid, uint64_t pd_addr) 461 { 462 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 463 464 /* wait for invalidate to complete */ 465 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0)); 466 amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST); 467 amdgpu_ring_write(ring, 0xff << 16); /* retry */ 468 amdgpu_ring_write(ring, 1 << vmid); /* mask */ 469 amdgpu_ring_write(ring, 0); /* value */ 470 amdgpu_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */ 471 } 472 473 static void si_dma_ring_emit_wreg(struct amdgpu_ring *ring, 474 uint32_t reg, uint32_t val) 475 { 476 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 477 amdgpu_ring_write(ring, (0xf << 16) | reg); 478 amdgpu_ring_write(ring, val); 479 } 480 481 static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = { 482 .copy_pte_num_dw = 5, 483 .copy_pte = si_dma_vm_copy_pte, 484 485 .write_pte = si_dma_vm_write_pte, 486 .set_pte_pde = si_dma_vm_set_pte_pde, 487 }; 488 489 static int si_dma_early_init(struct amdgpu_ip_block *ip_block) 490 { 491 struct amdgpu_device *adev = ip_block->adev; 492 493 adev->sdma.num_instances = SDMA_MAX_INSTANCE; 494 495 si_dma_set_ring_funcs(adev); 496 amdgpu_sdma_set_vm_pte_scheds(adev, &si_dma_vm_pte_funcs); 497 si_dma_set_irq_funcs(adev); 498 499 return 0; 500 } 501 502 static int si_dma_sw_init(struct amdgpu_ip_block *ip_block) 503 { 504 struct amdgpu_ring *ring; 505 int r, i; 506 struct amdgpu_device *adev = ip_block->adev; 507 508 /* DMA0 trap event */ 509 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224, 510 &adev->sdma.trap_irq); 511 if (r) 512 return r; 513 514 /* DMA1 trap event */ 515 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 244, 516 &adev->sdma.trap_irq); 517 if (r) 518 return r; 519 520 for (i = 0; i < adev->sdma.num_instances; i++) { 521 ring = &adev->sdma.instance[i].ring; 522 ring->ring_obj = NULL; 523 ring->use_doorbell = false; 524 sprintf(ring->name, "sdma%d", i); 525 r = amdgpu_ring_init(adev, ring, 1024, 526 &adev->sdma.trap_irq, 527 (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 : 528 AMDGPU_SDMA_IRQ_INSTANCE1, 529 AMDGPU_RING_PRIO_DEFAULT, NULL); 530 if (r) 531 return r; 532 } 533 534 return r; 535 } 536 537 static int si_dma_sw_fini(struct amdgpu_ip_block *ip_block) 538 { 539 struct amdgpu_device *adev = ip_block->adev; 540 int i; 541 542 for (i = 0; i < adev->sdma.num_instances; i++) 543 amdgpu_ring_fini(&adev->sdma.instance[i].ring); 544 545 return 0; 546 } 547 548 static int si_dma_hw_init(struct amdgpu_ip_block *ip_block) 549 { 550 struct amdgpu_device *adev = ip_block->adev; 551 int r; 552 553 r = si_dma_start(adev); 554 if (r) 555 return r; 556 si_dma_set_buffer_funcs(adev); 557 558 return 0; 559 } 560 561 static int si_dma_hw_fini(struct amdgpu_ip_block *ip_block) 562 { 563 si_dma_stop(ip_block->adev); 564 565 return 0; 566 } 567 568 static int si_dma_suspend(struct amdgpu_ip_block *ip_block) 569 { 570 return si_dma_hw_fini(ip_block); 571 } 572 573 static int si_dma_resume(struct amdgpu_ip_block *ip_block) 574 { 575 return si_dma_hw_init(ip_block); 576 } 577 578 static bool si_dma_is_idle(struct amdgpu_ip_block *ip_block) 579 { 580 struct amdgpu_device *adev = ip_block->adev; 581 582 u32 tmp = RREG32(mmSRBM_STATUS2); 583 584 if (tmp & (SRBM_STATUS2__DMA_BUSY_MASK | SRBM_STATUS2__DMA1_BUSY_MASK)) 585 return false; 586 587 return true; 588 } 589 590 static int si_dma_wait_for_idle(struct amdgpu_ip_block *ip_block) 591 { 592 unsigned i; 593 struct amdgpu_device *adev = ip_block->adev; 594 595 for (i = 0; i < adev->usec_timeout; i++) { 596 if (si_dma_is_idle(ip_block)) 597 return 0; 598 udelay(1); 599 } 600 return -ETIMEDOUT; 601 } 602 603 static int si_dma_set_trap_irq_state(struct amdgpu_device *adev, 604 struct amdgpu_irq_src *src, 605 unsigned type, 606 enum amdgpu_interrupt_state state) 607 { 608 u32 sdma_cntl; 609 610 switch (type) { 611 case AMDGPU_SDMA_IRQ_INSTANCE0: 612 switch (state) { 613 case AMDGPU_IRQ_STATE_DISABLE: 614 sdma_cntl = RREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET); 615 sdma_cntl &= ~DMA_CNTL__TRAP_ENABLE_MASK; 616 WREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl); 617 break; 618 case AMDGPU_IRQ_STATE_ENABLE: 619 sdma_cntl = RREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET); 620 sdma_cntl |= DMA_CNTL__TRAP_ENABLE_MASK; 621 WREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl); 622 break; 623 default: 624 break; 625 } 626 break; 627 case AMDGPU_SDMA_IRQ_INSTANCE1: 628 switch (state) { 629 case AMDGPU_IRQ_STATE_DISABLE: 630 sdma_cntl = RREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET); 631 sdma_cntl &= ~DMA_CNTL__TRAP_ENABLE_MASK; 632 WREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl); 633 break; 634 case AMDGPU_IRQ_STATE_ENABLE: 635 sdma_cntl = RREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET); 636 sdma_cntl |= DMA_CNTL__TRAP_ENABLE_MASK; 637 WREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl); 638 break; 639 default: 640 break; 641 } 642 break; 643 default: 644 break; 645 } 646 return 0; 647 } 648 649 static int si_dma_process_trap_irq(struct amdgpu_device *adev, 650 struct amdgpu_irq_src *source, 651 struct amdgpu_iv_entry *entry) 652 { 653 if (entry->src_id == 224) 654 amdgpu_fence_process(&adev->sdma.instance[0].ring); 655 else 656 amdgpu_fence_process(&adev->sdma.instance[1].ring); 657 return 0; 658 } 659 660 static int si_dma_set_clockgating_state(struct amdgpu_ip_block *ip_block, 661 enum amd_clockgating_state state) 662 { 663 u32 orig, data, offset; 664 int i; 665 bool enable; 666 struct amdgpu_device *adev = ip_block->adev; 667 668 enable = (state == AMD_CG_STATE_GATE); 669 670 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { 671 for (i = 0; i < adev->sdma.num_instances; i++) { 672 if (i == 0) 673 offset = DMA0_REGISTER_OFFSET; 674 else 675 offset = DMA1_REGISTER_OFFSET; 676 orig = data = RREG32(mmDMA_POWER_CNTL + offset); 677 data &= ~DMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; 678 if (data != orig) 679 WREG32(mmDMA_POWER_CNTL + offset, data); 680 WREG32(mmDMA_CLK_CTRL + offset, 0x00000100); 681 } 682 } else { 683 for (i = 0; i < adev->sdma.num_instances; i++) { 684 if (i == 0) 685 offset = DMA0_REGISTER_OFFSET; 686 else 687 offset = DMA1_REGISTER_OFFSET; 688 orig = data = RREG32(mmDMA_POWER_CNTL + offset); 689 data |= DMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; 690 if (data != orig) 691 WREG32(mmDMA_POWER_CNTL + offset, data); 692 693 orig = data = RREG32(mmDMA_CLK_CTRL + offset); 694 data = 0xff000000; 695 if (data != orig) 696 WREG32(mmDMA_CLK_CTRL + offset, data); 697 } 698 } 699 700 return 0; 701 } 702 703 static int si_dma_set_powergating_state(struct amdgpu_ip_block *ip_block, 704 enum amd_powergating_state state) 705 { 706 u32 tmp; 707 708 struct amdgpu_device *adev = ip_block->adev; 709 710 WREG32(mmDMA_PGFSM_WRITE, 0x00002000); 711 WREG32(mmDMA_PGFSM_CONFIG, 0x100010ff); 712 713 for (tmp = 0; tmp < 5; tmp++) 714 WREG32(mmDMA_PGFSM_WRITE, 0); 715 716 return 0; 717 } 718 719 static const struct amd_ip_funcs si_dma_ip_funcs = { 720 .name = "si_dma", 721 .early_init = si_dma_early_init, 722 .sw_init = si_dma_sw_init, 723 .sw_fini = si_dma_sw_fini, 724 .hw_init = si_dma_hw_init, 725 .hw_fini = si_dma_hw_fini, 726 .suspend = si_dma_suspend, 727 .resume = si_dma_resume, 728 .wait_for_idle = si_dma_wait_for_idle, 729 .set_clockgating_state = si_dma_set_clockgating_state, 730 .set_powergating_state = si_dma_set_powergating_state, 731 }; 732 733 static const struct amdgpu_ring_funcs si_dma_ring_funcs = { 734 .type = AMDGPU_RING_TYPE_SDMA, 735 .align_mask = 0xf, 736 .nop = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0), 737 .support_64bit_ptrs = false, 738 .get_rptr = si_dma_ring_get_rptr, 739 .get_wptr = si_dma_ring_get_wptr, 740 .set_wptr = si_dma_ring_set_wptr, 741 .emit_frame_size = 742 3 + 3 + /* hdp flush / invalidate */ 743 6 + /* si_dma_ring_emit_pipeline_sync */ 744 SI_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* si_dma_ring_emit_vm_flush */ 745 9 + 9 + 9, /* si_dma_ring_emit_fence x3 for user fence, vm fence */ 746 .emit_ib_size = 7 + 3, /* si_dma_ring_emit_ib */ 747 .emit_ib = si_dma_ring_emit_ib, 748 .emit_fence = si_dma_ring_emit_fence, 749 .emit_pipeline_sync = si_dma_ring_emit_pipeline_sync, 750 .emit_vm_flush = si_dma_ring_emit_vm_flush, 751 .test_ring = si_dma_ring_test_ring, 752 .test_ib = si_dma_ring_test_ib, 753 .insert_nop = amdgpu_ring_insert_nop, 754 .pad_ib = si_dma_ring_pad_ib, 755 .emit_wreg = si_dma_ring_emit_wreg, 756 }; 757 758 static void si_dma_set_ring_funcs(struct amdgpu_device *adev) 759 { 760 int i; 761 762 for (i = 0; i < adev->sdma.num_instances; i++) 763 adev->sdma.instance[i].ring.funcs = &si_dma_ring_funcs; 764 } 765 766 static const struct amdgpu_irq_src_funcs si_dma_trap_irq_funcs = { 767 .set = si_dma_set_trap_irq_state, 768 .process = si_dma_process_trap_irq, 769 }; 770 771 static void si_dma_set_irq_funcs(struct amdgpu_device *adev) 772 { 773 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST; 774 adev->sdma.trap_irq.funcs = &si_dma_trap_irq_funcs; 775 } 776 777 /** 778 * si_dma_emit_copy_buffer - copy buffer using the sDMA engine 779 * 780 * @ib: indirect buffer to copy to 781 * @src_offset: src GPU address 782 * @dst_offset: dst GPU address 783 * @byte_count: number of bytes to xfer 784 * @copy_flags: unused 785 * 786 * Copy GPU buffers using the DMA engine (VI). 787 * Used by the amdgpu ttm implementation to move pages if 788 * registered as the asic copy callback. 789 */ 790 static void si_dma_emit_copy_buffer(struct amdgpu_ib *ib, 791 uint64_t src_offset, 792 uint64_t dst_offset, 793 uint32_t byte_count, 794 uint32_t copy_flags) 795 { 796 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY, 797 1, 0, 0, byte_count); 798 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); 799 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); 800 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) & 0xff; 801 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset) & 0xff; 802 } 803 804 /** 805 * si_dma_emit_fill_buffer - fill buffer using the sDMA engine 806 * 807 * @ib: indirect buffer to copy to 808 * @src_data: value to write to buffer 809 * @dst_offset: dst GPU address 810 * @byte_count: number of bytes to xfer 811 * 812 * Fill GPU buffers using the DMA engine (VI). 813 */ 814 static void si_dma_emit_fill_buffer(struct amdgpu_ib *ib, 815 uint32_t src_data, 816 uint64_t dst_offset, 817 uint32_t byte_count) 818 { 819 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_CONSTANT_FILL, 820 0, 0, 0, byte_count / 4); 821 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); 822 ib->ptr[ib->length_dw++] = src_data; 823 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) << 16; 824 } 825 826 827 static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = { 828 .copy_max_bytes = 0xffff8, 829 .copy_num_dw = 5, 830 .emit_copy_buffer = si_dma_emit_copy_buffer, 831 832 .fill_max_bytes = 0xffff8, 833 .fill_num_dw = 4, 834 .emit_fill_buffer = si_dma_emit_fill_buffer, 835 }; 836 837 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev) 838 { 839 amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs); 840 } 841 842 const struct amdgpu_ip_block_version si_dma_ip_block = 843 { 844 .type = AMD_IP_BLOCK_TYPE_SDMA, 845 .major = 1, 846 .minor = 0, 847 .rev = 0, 848 .funcs = &si_dma_ip_funcs, 849 }; 850