Lines Matching +full:gfx +full:- +full:mem
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
27 #include <linux/io-64-nonatomic-lo-hi.h>
45 return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev); in amdgpu_gmc_is_pdb0_enabled()
49 * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0
60 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; in amdgpu_gmc_pdb0_alloc()
61 uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21; in amdgpu_gmc_pdb0_alloc()
62 uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift; in amdgpu_gmc_pdb0_alloc()
74 r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo); in amdgpu_gmc_pdb0_alloc()
78 r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false); in amdgpu_gmc_pdb0_alloc()
82 r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM); in amdgpu_gmc_pdb0_alloc()
85 r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0); in amdgpu_gmc_pdb0_alloc()
89 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); in amdgpu_gmc_pdb0_alloc()
93 amdgpu_bo_unpin(adev->gmc.pdb0_bo); in amdgpu_gmc_pdb0_alloc()
95 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); in amdgpu_gmc_pdb0_alloc()
97 amdgpu_bo_unref(&adev->gmc.pdb0_bo); in amdgpu_gmc_pdb0_alloc()
102 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO
114 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_gmc_get_pde_for_bo()
116 switch (bo->tbo.resource->mem_type) { in amdgpu_gmc_get_pde_for_bo()
118 *addr = bo->tbo.ttm->dma_address[0]; in amdgpu_gmc_get_pde_for_bo()
127 *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, bo->tbo.resource); in amdgpu_gmc_get_pde_for_bo()
132 * amdgpu_gmc_pd_addr - return the address of the root directory
136 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_gmc_pd_addr()
140 if (adev->asic_type >= CHIP_VEGA10) { in amdgpu_gmc_pd_addr()
143 amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags); in amdgpu_gmc_pd_addr()
152 * amdgpu_gmc_set_pte_pde - update the page tables using CPU
180 * amdgpu_gmc_agp_addr - return the address in the AGP address space
189 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_gmc_agp_addr()
191 if (!bo->ttm) in amdgpu_gmc_agp_addr()
194 if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) in amdgpu_gmc_agp_addr()
197 if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) in amdgpu_gmc_agp_addr()
200 return adev->gmc.agp_start + bo->ttm->dma_address[0]; in amdgpu_gmc_agp_addr()
204 * amdgpu_gmc_vram_location - try to find VRAM location
219 mc->vram_start = base; in amdgpu_gmc_vram_location()
220 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; in amdgpu_gmc_vram_location()
221 if (limit < mc->real_vram_size) in amdgpu_gmc_vram_location()
222 mc->real_vram_size = limit; in amdgpu_gmc_vram_location()
224 if (vis_limit && vis_limit < mc->visible_vram_size) in amdgpu_gmc_vram_location()
225 mc->visible_vram_size = vis_limit; in amdgpu_gmc_vram_location()
227 if (mc->real_vram_size < mc->visible_vram_size) in amdgpu_gmc_vram_location()
228 mc->visible_vram_size = mc->real_vram_size; in amdgpu_gmc_vram_location()
230 if (mc->xgmi.num_physical_nodes == 0) { in amdgpu_gmc_vram_location()
231 mc->fb_start = mc->vram_start; in amdgpu_gmc_vram_location()
232 mc->fb_end = mc->vram_end; in amdgpu_gmc_vram_location()
234 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", in amdgpu_gmc_vram_location()
235 mc->mc_vram_size >> 20, mc->vram_start, in amdgpu_gmc_vram_location()
236 mc->vram_end, mc->real_vram_size >> 20); in amdgpu_gmc_vram_location()
239 /** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture
258 u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1; in amdgpu_gmc_sysvm_location()
259 mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id; in amdgpu_gmc_sysvm_location()
260 mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1; in amdgpu_gmc_sysvm_location()
262 mc->gart_start = ALIGN(hive_vram_end + 1, four_gb); in amdgpu_gmc_sysvm_location()
263 mc->gart_end = mc->gart_start + mc->gart_size - 1; in amdgpu_gmc_sysvm_location()
265 /* set mc->vram_start to 0 to switch the returned GPU address of in amdgpu_gmc_sysvm_location()
268 mc->vram_start = 0; in amdgpu_gmc_sysvm_location()
269 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; in amdgpu_gmc_sysvm_location()
270 mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size); in amdgpu_gmc_sysvm_location()
272 mc->fb_start = hive_vram_start; in amdgpu_gmc_sysvm_location()
273 mc->fb_end = hive_vram_end; in amdgpu_gmc_sysvm_location()
275 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", in amdgpu_gmc_sysvm_location()
276 mc->mc_vram_size >> 20, mc->vram_start, in amdgpu_gmc_sysvm_location()
277 mc->vram_end, mc->real_vram_size >> 20); in amdgpu_gmc_sysvm_location()
278 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", in amdgpu_gmc_sysvm_location()
279 mc->gart_size >> 20, mc->gart_start, mc->gart_end); in amdgpu_gmc_sysvm_location()
283 * amdgpu_gmc_gart_location - try to find GART location
298 u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1); in amdgpu_gmc_gart_location()
303 size_bf = mc->fb_start; in amdgpu_gmc_gart_location()
304 size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb); in amdgpu_gmc_gart_location()
306 if (mc->gart_size > max(size_bf, size_af)) { in amdgpu_gmc_gart_location()
307 dev_warn(adev->dev, "limiting GART\n"); in amdgpu_gmc_gart_location()
308 mc->gart_size = max(size_bf, size_af); in amdgpu_gmc_gart_location()
313 mc->gart_start = max_mc_address - mc->gart_size + 1; in amdgpu_gmc_gart_location()
316 mc->gart_start = 0; in amdgpu_gmc_gart_location()
320 if ((size_bf >= mc->gart_size && size_bf < size_af) || in amdgpu_gmc_gart_location()
321 (size_af < mc->gart_size)) in amdgpu_gmc_gart_location()
322 mc->gart_start = 0; in amdgpu_gmc_gart_location()
324 mc->gart_start = max_mc_address - mc->gart_size + 1; in amdgpu_gmc_gart_location()
328 mc->gart_start &= ~(four_gb - 1); in amdgpu_gmc_gart_location()
329 mc->gart_end = mc->gart_start + mc->gart_size - 1; in amdgpu_gmc_gart_location()
330 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", in amdgpu_gmc_gart_location()
331 mc->gart_size >> 20, mc->gart_start, mc->gart_end); in amdgpu_gmc_gart_location()
335 * amdgpu_gmc_agp_location - try to find AGP location
348 const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1); in amdgpu_gmc_agp_location()
351 if (mc->fb_start > mc->gart_start) { in amdgpu_gmc_agp_location()
352 size_bf = (mc->fb_start & sixteen_gb_mask) - in amdgpu_gmc_agp_location()
353 ALIGN(mc->gart_end + 1, sixteen_gb); in amdgpu_gmc_agp_location()
354 size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb); in amdgpu_gmc_agp_location()
356 size_bf = mc->fb_start & sixteen_gb_mask; in amdgpu_gmc_agp_location()
357 size_af = (mc->gart_start & sixteen_gb_mask) - in amdgpu_gmc_agp_location()
358 ALIGN(mc->fb_end + 1, sixteen_gb); in amdgpu_gmc_agp_location()
362 mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask; in amdgpu_gmc_agp_location()
363 mc->agp_size = size_bf; in amdgpu_gmc_agp_location()
365 mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb); in amdgpu_gmc_agp_location()
366 mc->agp_size = size_af; in amdgpu_gmc_agp_location()
369 mc->agp_end = mc->agp_start + mc->agp_size - 1; in amdgpu_gmc_agp_location()
370 dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n", in amdgpu_gmc_agp_location()
371 mc->agp_size >> 20, mc->agp_start, mc->agp_end); in amdgpu_gmc_agp_location()
375 * amdgpu_gmc_set_agp_default - Set the default AGP aperture value.
388 mc->agp_start = 0xffffffffffff; in amdgpu_gmc_set_agp_default()
389 mc->agp_end = 0; in amdgpu_gmc_set_agp_default()
390 mc->agp_size = 0; in amdgpu_gmc_set_agp_default()
394 * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid
405 * amdgpu_gmc_filter_faults - filter VM faults
421 struct amdgpu_gmc *gmc = &adev->gmc; in amdgpu_gmc_filter_faults()
427 if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp)) in amdgpu_gmc_filter_faults()
431 stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) - in amdgpu_gmc_filter_faults()
433 if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp) in amdgpu_gmc_filter_faults()
438 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; in amdgpu_gmc_filter_faults()
439 while (fault->timestamp >= stamp) { in amdgpu_gmc_filter_faults()
442 if (atomic64_read(&fault->key) == key) { in amdgpu_gmc_filter_faults()
450 if (fault->timestamp_expiry != 0 && in amdgpu_gmc_filter_faults()
451 amdgpu_ih_ts_after(fault->timestamp_expiry, in amdgpu_gmc_filter_faults()
458 tmp = fault->timestamp; in amdgpu_gmc_filter_faults()
459 fault = &gmc->fault_ring[fault->next]; in amdgpu_gmc_filter_faults()
462 if (fault->timestamp >= tmp) in amdgpu_gmc_filter_faults()
467 fault = &gmc->fault_ring[gmc->last_fault]; in amdgpu_gmc_filter_faults()
468 atomic64_set(&fault->key, key); in amdgpu_gmc_filter_faults()
469 fault->timestamp = timestamp; in amdgpu_gmc_filter_faults()
472 fault->next = gmc->fault_hash[hash].idx; in amdgpu_gmc_filter_faults()
473 gmc->fault_hash[hash].idx = gmc->last_fault++; in amdgpu_gmc_filter_faults()
478 * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter
490 struct amdgpu_gmc *gmc = &adev->gmc; in amdgpu_gmc_filter_faults_remove()
499 if (adev->irq.retry_cam_enabled) in amdgpu_gmc_filter_faults_remove()
502 ih = &adev->irq.ih1; in amdgpu_gmc_filter_faults_remove()
508 last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1); in amdgpu_gmc_filter_faults_remove()
511 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; in amdgpu_gmc_filter_faults_remove()
513 if (atomic64_read(&fault->key) == key) { in amdgpu_gmc_filter_faults_remove()
518 fault->timestamp_expiry = last_ts; in amdgpu_gmc_filter_faults_remove()
522 tmp = fault->timestamp; in amdgpu_gmc_filter_faults_remove()
523 fault = &gmc->fault_ring[fault->next]; in amdgpu_gmc_filter_faults_remove()
524 } while (fault->timestamp < tmp); in amdgpu_gmc_filter_faults_remove()
595 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { in amdgpu_gmc_allocate_vm_inv_eng()
598 if (adev->enable_mes) in amdgpu_gmc_allocate_vm_inv_eng()
601 if (adev->enable_uni_mes) in amdgpu_gmc_allocate_vm_inv_eng()
604 if (adev->enable_umsch_mm) in amdgpu_gmc_allocate_vm_inv_eng()
608 for (i = 0; i < adev->num_rings; ++i) { in amdgpu_gmc_allocate_vm_inv_eng()
609 ring = adev->rings[i]; in amdgpu_gmc_allocate_vm_inv_eng()
610 vmhub = ring->vm_hub; in amdgpu_gmc_allocate_vm_inv_eng()
612 if (ring == &adev->mes.ring[0] || in amdgpu_gmc_allocate_vm_inv_eng()
613 ring == &adev->mes.ring[1] || in amdgpu_gmc_allocate_vm_inv_eng()
614 ring == &adev->umsch_mm.ring || in amdgpu_gmc_allocate_vm_inv_eng()
615 ring == &adev->cper.ring_buf) in amdgpu_gmc_allocate_vm_inv_eng()
624 dev_err(adev->dev, "no VM inv eng for ring %s\n", in amdgpu_gmc_allocate_vm_inv_eng()
625 ring->name); in amdgpu_gmc_allocate_vm_inv_eng()
626 return -EINVAL; in amdgpu_gmc_allocate_vm_inv_eng()
629 ring->vm_inv_eng = inv_eng - 1; in amdgpu_gmc_allocate_vm_inv_eng()
630 vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng); in amdgpu_gmc_allocate_vm_inv_eng()
632 dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", in amdgpu_gmc_allocate_vm_inv_eng()
633 ring->name, ring->vm_inv_eng, ring->vm_hub); in amdgpu_gmc_allocate_vm_inv_eng()
638 * engine with the SDMA gfx ring. This change ensures efficient in amdgpu_gmc_allocate_vm_inv_eng()
644 shared_ring->vm_inv_eng = ring->vm_inv_eng; in amdgpu_gmc_allocate_vm_inv_eng()
645 dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n", in amdgpu_gmc_allocate_vm_inv_eng()
646 ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub); in amdgpu_gmc_allocate_vm_inv_eng()
657 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_gmc_flush_gpu_tlb()
658 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; in amdgpu_gmc_flush_gpu_tlb()
663 if (!hub->sdma_invalidation_workaround || vmid || in amdgpu_gmc_flush_gpu_tlb()
664 !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || in amdgpu_gmc_flush_gpu_tlb()
665 !ring->sched.ready) { in amdgpu_gmc_flush_gpu_tlb()
670 if (!down_read_trylock(&adev->reset_domain->sem)) in amdgpu_gmc_flush_gpu_tlb()
673 if (adev->gmc.flush_tlb_needs_extra_type_2) in amdgpu_gmc_flush_gpu_tlb()
674 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, in amdgpu_gmc_flush_gpu_tlb()
677 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) in amdgpu_gmc_flush_gpu_tlb()
678 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, in amdgpu_gmc_flush_gpu_tlb()
681 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub, in amdgpu_gmc_flush_gpu_tlb()
683 up_read(&adev->reset_domain->sem); in amdgpu_gmc_flush_gpu_tlb()
692 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_gmc_flush_gpu_tlb()
693 r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.high_pr, in amdgpu_gmc_flush_gpu_tlb()
700 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); in amdgpu_gmc_flush_gpu_tlb()
701 job->vm_needs_flush = true; in amdgpu_gmc_flush_gpu_tlb()
702 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop; in amdgpu_gmc_flush_gpu_tlb()
703 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_gmc_flush_gpu_tlb()
705 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_gmc_flush_gpu_tlb()
713 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_gmc_flush_gpu_tlb()
714 dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); in amdgpu_gmc_flush_gpu_tlb()
721 struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; in amdgpu_gmc_flush_gpu_tlb_pasid()
722 struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; in amdgpu_gmc_flush_gpu_tlb_pasid()
731 if (!down_read_trylock(&adev->reset_domain->sem)) in amdgpu_gmc_flush_gpu_tlb_pasid()
734 if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { in amdgpu_gmc_flush_gpu_tlb_pasid()
735 if (adev->gmc.flush_tlb_needs_extra_type_2) in amdgpu_gmc_flush_gpu_tlb_pasid()
736 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, in amdgpu_gmc_flush_gpu_tlb_pasid()
740 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) in amdgpu_gmc_flush_gpu_tlb_pasid()
741 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, in amdgpu_gmc_flush_gpu_tlb_pasid()
745 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, in amdgpu_gmc_flush_gpu_tlb_pasid()
751 ndw = kiq->pmf->invalidate_tlbs_size + 8; in amdgpu_gmc_flush_gpu_tlb_pasid()
753 if (adev->gmc.flush_tlb_needs_extra_type_2) in amdgpu_gmc_flush_gpu_tlb_pasid()
754 ndw += kiq->pmf->invalidate_tlbs_size; in amdgpu_gmc_flush_gpu_tlb_pasid()
756 if (adev->gmc.flush_tlb_needs_extra_type_0) in amdgpu_gmc_flush_gpu_tlb_pasid()
757 ndw += kiq->pmf->invalidate_tlbs_size; in amdgpu_gmc_flush_gpu_tlb_pasid()
759 spin_lock(&adev->gfx.kiq[inst].ring_lock); in amdgpu_gmc_flush_gpu_tlb_pasid()
762 spin_unlock(&adev->gfx.kiq[inst].ring_lock); in amdgpu_gmc_flush_gpu_tlb_pasid()
765 if (adev->gmc.flush_tlb_needs_extra_type_2) in amdgpu_gmc_flush_gpu_tlb_pasid()
766 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); in amdgpu_gmc_flush_gpu_tlb_pasid()
768 if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) in amdgpu_gmc_flush_gpu_tlb_pasid()
769 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); in amdgpu_gmc_flush_gpu_tlb_pasid()
771 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); in amdgpu_gmc_flush_gpu_tlb_pasid()
775 spin_unlock(&adev->gfx.kiq[inst].ring_lock); in amdgpu_gmc_flush_gpu_tlb_pasid()
780 spin_unlock(&adev->gfx.kiq[inst].ring_lock); in amdgpu_gmc_flush_gpu_tlb_pasid()
786 !amdgpu_reset_pending(adev->reset_domain)) { in amdgpu_gmc_flush_gpu_tlb_pasid()
792 dev_err(adev->dev, "timeout waiting for kiq fence\n"); in amdgpu_gmc_flush_gpu_tlb_pasid()
793 r = -ETIME; in amdgpu_gmc_flush_gpu_tlb_pasid()
799 up_read(&adev->reset_domain->sem); in amdgpu_gmc_flush_gpu_tlb_pasid()
808 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst]; in amdgpu_gmc_fw_reg_write_reg_wait()
809 struct amdgpu_ring *ring = &kiq->ring; in amdgpu_gmc_fw_reg_write_reg_wait()
814 if (adev->mes.ring[0].sched.ready) { in amdgpu_gmc_fw_reg_write_reg_wait()
820 spin_lock_irqsave(&kiq->ring_lock, flags); in amdgpu_gmc_fw_reg_write_reg_wait()
829 spin_unlock_irqrestore(&kiq->ring_lock, flags); in amdgpu_gmc_fw_reg_write_reg_wait()
839 !amdgpu_reset_pending(adev->reset_domain)) { in amdgpu_gmc_fw_reg_write_reg_wait()
852 spin_unlock_irqrestore(&kiq->ring_lock, flags); in amdgpu_gmc_fw_reg_write_reg_wait()
854 dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1); in amdgpu_gmc_fw_reg_write_reg_wait()
858 * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ
877 adev->gmc.tmz_enabled = false; in amdgpu_gmc_tmz_set()
878 dev_info(adev->dev, in amdgpu_gmc_tmz_set()
881 adev->gmc.tmz_enabled = true; in amdgpu_gmc_tmz_set()
882 dev_info(adev->dev, in amdgpu_gmc_tmz_set()
907 adev->gmc.tmz_enabled = false; in amdgpu_gmc_tmz_set()
908 dev_info(adev->dev, in amdgpu_gmc_tmz_set()
911 adev->gmc.tmz_enabled = true; in amdgpu_gmc_tmz_set()
912 dev_info(adev->dev, in amdgpu_gmc_tmz_set()
917 adev->gmc.tmz_enabled = false; in amdgpu_gmc_tmz_set()
918 dev_info(adev->dev, in amdgpu_gmc_tmz_set()
925 * amdgpu_gmc_noretry_set -- set per asic noretry defaults
928 * Set a per asic default for the no-retry parameter.
933 struct amdgpu_gmc *gmc = &adev->gmc; in amdgpu_gmc_noretry_set()
945 gmc->noretry = 1; in amdgpu_gmc_noretry_set()
947 gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry; in amdgpu_gmc_noretry_set()
956 hub = &adev->vmhub[hub_type]; in amdgpu_gmc_set_vm_fault_masks()
958 reg = hub->vm_context0_cntl + hub->ctx_distance * i; in amdgpu_gmc_set_vm_fault_masks()
965 tmp |= hub->vm_cntx_cntl_vm_fault; in amdgpu_gmc_set_vm_fault_masks()
967 tmp &= ~hub->vm_cntx_cntl_vm_fault; in amdgpu_gmc_set_vm_fault_masks()
983 adev->mman.stolen_reserved_offset = 0; in amdgpu_gmc_get_vbios_allocations()
984 adev->mman.stolen_reserved_size = 0; in amdgpu_gmc_get_vbios_allocations()
994 switch (adev->asic_type) { in amdgpu_gmc_get_vbios_allocations()
996 adev->mman.keep_stolen_vga_memory = true; in amdgpu_gmc_get_vbios_allocations()
1002 adev->mman.stolen_reserved_offset = 0x500000; in amdgpu_gmc_get_vbios_allocations()
1003 adev->mman.stolen_reserved_size = 0x200000; in amdgpu_gmc_get_vbios_allocations()
1009 adev->mman.keep_stolen_vga_memory = true; in amdgpu_gmc_get_vbios_allocations()
1012 adev->mman.keep_stolen_vga_memory = false; in amdgpu_gmc_get_vbios_allocations()
1022 if (adev->mman.keep_stolen_vga_memory) in amdgpu_gmc_get_vbios_allocations()
1026 /* set to 0 if the pre-OS buffer uses up most of vram */ in amdgpu_gmc_get_vbios_allocations()
1027 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) in amdgpu_gmc_get_vbios_allocations()
1031 adev->mman.stolen_vga_size = AMDGPU_VBIOS_VGA_ALLOCATION; in amdgpu_gmc_get_vbios_allocations()
1032 adev->mman.stolen_extended_size = size - adev->mman.stolen_vga_size; in amdgpu_gmc_get_vbios_allocations()
1034 adev->mman.stolen_vga_size = size; in amdgpu_gmc_get_vbios_allocations()
1035 adev->mman.stolen_extended_size = 0; in amdgpu_gmc_get_vbios_allocations()
1040 * amdgpu_gmc_init_pdb0 - initialize PDB0
1046 * a 2-level system VM page table: PDB0->PTB, to cover both
1058 uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW? in amdgpu_gmc_init_pdb0()
1061 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; in amdgpu_gmc_init_pdb0()
1062 u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21; in amdgpu_gmc_init_pdb0()
1064 u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo); in amdgpu_gmc_init_pdb0()
1073 flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1)); in amdgpu_gmc_init_pdb0()
1076 vram_addr = adev->vm_manager.vram_base_offset; in amdgpu_gmc_init_pdb0()
1078 vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; in amdgpu_gmc_init_pdb0()
1085 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags); in amdgpu_gmc_init_pdb0()
1094 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags); in amdgpu_gmc_init_pdb0()
1099 * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC
1107 return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset; in amdgpu_gmc_vram_mc2pa()
1111 * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from
1152 ret = -EIO; in amdgpu_gmc_vram_checking()
1158 ret = -EIO; in amdgpu_gmc_vram_checking()
1162 ret = memcmp(vram_ptr + size - 10, cptr, 10); in amdgpu_gmc_vram_checking()
1164 ret = -EIO; in amdgpu_gmc_vram_checking()
1193 for_each_inst(mode, adev->gmc.supported_nps_modes) { in available_memory_partition_show()
1213 for_each_inst(i, adev->gmc.supported_nps_modes) { in current_memory_partition_store()
1221 return -EINVAL; in current_memory_partition_store()
1223 if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) { in current_memory_partition_store()
1225 adev->dev, in current_memory_partition_store()
1235 atomic_set(&hive->requested_nps_mode, mode); in current_memory_partition_store()
1238 adev->gmc.requested_nps_mode = mode; in current_memory_partition_store()
1242 adev->dev, in current_memory_partition_store()
1257 return -EPERM; in current_memory_partition_show()
1259 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); in current_memory_partition_show()
1275 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) in amdgpu_gmc_sysfs_init()
1278 nps_switch_support = (hweight32(adev->gmc.supported_nps_modes & in amdgpu_gmc_sysfs_init()
1284 r = device_create_file(adev->dev, in amdgpu_gmc_sysfs_init()
1290 return device_create_file(adev->dev, in amdgpu_gmc_sysfs_init()
1296 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) in amdgpu_gmc_sysfs_fini()
1299 device_remove_file(adev->dev, &dev_attr_current_memory_partition); in amdgpu_gmc_sysfs_fini()
1300 device_remove_file(adev->dev, &dev_attr_available_memory_partition); in amdgpu_gmc_sysfs_fini()
1313 return -EINVAL; in amdgpu_gmc_get_nps_memranges()
1315 refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) && in amdgpu_gmc_get_nps_memranges()
1316 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS); in amdgpu_gmc_get_nps_memranges()
1328 adev->dev, in amdgpu_gmc_get_nps_memranges()
1329 "NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d", in amdgpu_gmc_get_nps_memranges()
1331 ret = -EINVAL; in amdgpu_gmc_get_nps_memranges()
1338 adev->dev, in amdgpu_gmc_get_nps_memranges()
1339 "Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx", in amdgpu_gmc_get_nps_memranges()
1342 ret = -EINVAL; in amdgpu_gmc_get_nps_memranges()
1347 for (j = i - 1; j >= 0; j--) { in amdgpu_gmc_get_nps_memranges()
1353 adev->dev, in amdgpu_gmc_get_nps_memranges()
1354 "overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]", in amdgpu_gmc_get_nps_memranges()
1359 ret = -EINVAL; in amdgpu_gmc_get_nps_memranges()
1365 (ranges[i].base_address - in amdgpu_gmc_get_nps_memranges()
1366 adev->vm_manager.vram_base_offset) >> in amdgpu_gmc_get_nps_memranges()
1369 (ranges[i].limit_address - in amdgpu_gmc_get_nps_memranges()
1370 adev->vm_manager.vram_base_offset) >> in amdgpu_gmc_get_nps_memranges()
1373 ranges[i].limit_address - ranges[i].base_address + 1; in amdgpu_gmc_get_nps_memranges()
1388 if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU)) in amdgpu_gmc_request_memory_partition()
1389 return -EOPNOTSUPP; in amdgpu_gmc_request_memory_partition()
1391 if (!adev->psp.funcs) { in amdgpu_gmc_request_memory_partition()
1392 dev_err(adev->dev, in amdgpu_gmc_request_memory_partition()
1394 return -EINVAL; in amdgpu_gmc_request_memory_partition()
1397 return psp_memory_partition(&adev->psp, nps_mode); in amdgpu_gmc_request_memory_partition()
1404 return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) == in amdgpu_gmc_need_nps_switch_req()
1414 if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes || in amdgpu_gmc_prepare_nps_mode_change()
1415 !adev->gmc.gmc_funcs->request_mem_partition_mode) in amdgpu_gmc_prepare_nps_mode_change()
1418 cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); in amdgpu_gmc_prepare_nps_mode_change()
1421 req_nps_mode = atomic_read(&hive->requested_nps_mode); in amdgpu_gmc_prepare_nps_mode_change()
1432 req_nps_mode = adev->gmc.requested_nps_mode; in amdgpu_gmc_prepare_nps_mode_change()
1437 r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode); in amdgpu_gmc_prepare_nps_mode_change()
1440 dev_err(adev->dev, "NPS mode change request failed\n"); in amdgpu_gmc_prepare_nps_mode_change()
1443 adev->dev, in amdgpu_gmc_prepare_nps_mode_change()
1449 if (adev->gmc.gmc_funcs->need_reset_on_init) in amdgpu_gmc_need_reset_on_init()
1450 return adev->gmc.gmc_funcs->need_reset_on_init(adev); in amdgpu_gmc_need_reset_on_init()
1458 switch (adev->gmc.num_mem_partitions) { in amdgpu_gmc_get_vf_memory_partition()
1479 if (adev->nbio.funcs && in amdgpu_gmc_get_memory_partition()
1480 adev->nbio.funcs->get_memory_partition_mode) in amdgpu_gmc_get_memory_partition()
1481 mode = adev->nbio.funcs->get_memory_partition_mode(adev, in amdgpu_gmc_get_memory_partition()
1484 dev_warn(adev->dev, "memory partition mode query is not supported\n"); in amdgpu_gmc_get_memory_partition()
1508 !(BIT(mode - 1) & supp_modes)) in amdgpu_gmc_validate_partition_info()
1514 valid = (adev->gmc.num_mem_partitions == 1); in amdgpu_gmc_validate_partition_info()
1517 valid = (adev->gmc.num_mem_partitions == 2); in amdgpu_gmc_validate_partition_info()
1520 valid = (adev->gmc.num_mem_partitions == 3 || in amdgpu_gmc_validate_partition_info()
1521 adev->gmc.num_mem_partitions == 4); in amdgpu_gmc_validate_partition_info()
1524 valid = (adev->gmc.num_mem_partitions == 8); in amdgpu_gmc_validate_partition_info()
1555 num_xcc = NUM_XCC(adev->gfx.xcc_mask); in amdgpu_gmc_init_acpi_mem_ranges()
1556 xcc_mask = (1U << num_xcc) - 1; in amdgpu_gmc_init_acpi_mem_ranges()
1580 adev->gmc.num_mem_partitions = num_ranges; in amdgpu_gmc_init_acpi_mem_ranges()
1594 adev->gmc.num_mem_partitions = 0; in amdgpu_gmc_init_sw_mem_ranges()
1597 adev->gmc.num_mem_partitions = 1; in amdgpu_gmc_init_sw_mem_ranges()
1600 adev->gmc.num_mem_partitions = 2; in amdgpu_gmc_init_sw_mem_ranges()
1603 if (adev->flags & AMD_IS_APU) in amdgpu_gmc_init_sw_mem_ranges()
1604 adev->gmc.num_mem_partitions = 3; in amdgpu_gmc_init_sw_mem_ranges()
1606 adev->gmc.num_mem_partitions = 4; in amdgpu_gmc_init_sw_mem_ranges()
1609 adev->gmc.num_mem_partitions = 8; in amdgpu_gmc_init_sw_mem_ranges()
1612 adev->gmc.num_mem_partitions = 1; in amdgpu_gmc_init_sw_mem_ranges()
1618 &adev->gmc.num_mem_partitions); in amdgpu_gmc_init_sw_mem_ranges()
1621 for (i = 1; i < adev->gmc.num_mem_partitions; ++i) { in amdgpu_gmc_init_sw_mem_ranges()
1623 mem_ranges[i - 1].range.lpfn) in amdgpu_gmc_init_sw_mem_ranges()
1628 if (!adev->gmc.num_mem_partitions) { in amdgpu_gmc_init_sw_mem_ranges()
1629 dev_warn(adev->dev, in amdgpu_gmc_init_sw_mem_ranges()
1631 adev->gmc.num_mem_partitions = 1; in amdgpu_gmc_init_sw_mem_ranges()
1634 size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT; in amdgpu_gmc_init_sw_mem_ranges()
1635 size /= adev->gmc.num_mem_partitions; in amdgpu_gmc_init_sw_mem_ranges()
1637 for (i = 0; i < adev->gmc.num_mem_partitions; ++i) { in amdgpu_gmc_init_sw_mem_ranges()
1641 mem_ranges[i].range.lpfn = start_addr + size - 1; in amdgpu_gmc_init_sw_mem_ranges()
1645 l = adev->gmc.num_mem_partitions - 1; in amdgpu_gmc_init_sw_mem_ranges()
1650 (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1; in amdgpu_gmc_init_sw_mem_ranges()
1652 adev->gmc.real_vram_size - in amdgpu_gmc_init_sw_mem_ranges()
1660 adev->gmc.mem_partitions = kcalloc(AMDGPU_MAX_MEM_RANGES, in amdgpu_gmc_init_mem_ranges()
1663 if (!adev->gmc.mem_partitions) in amdgpu_gmc_init_mem_ranges()
1664 return -ENOMEM; in amdgpu_gmc_init_mem_ranges()
1666 if (adev->gmc.is_app_apu) in amdgpu_gmc_init_mem_ranges()
1667 amdgpu_gmc_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions); in amdgpu_gmc_init_mem_ranges()
1669 amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions); in amdgpu_gmc_init_mem_ranges()
1677 dev_warn(adev->dev, in amdgpu_gmc_init_mem_ranges()
1678 "Mem ranges not matching with hardware config\n"); in amdgpu_gmc_init_mem_ranges()