1d38ceaf9SAlex Deucher /* 2d38ceaf9SAlex Deucher * Copyright 2008 Advanced Micro Devices, Inc. 3d38ceaf9SAlex Deucher * Copyright 2008 Red Hat Inc. 4d38ceaf9SAlex Deucher * Copyright 2009 Jerome Glisse. 5d38ceaf9SAlex Deucher * 6d38ceaf9SAlex Deucher * Permission is hereby granted, free of charge, to any person obtaining a 7d38ceaf9SAlex Deucher * copy of this software and associated documentation files (the "Software"), 8d38ceaf9SAlex Deucher * to deal in the Software without restriction, including without limitation 9d38ceaf9SAlex Deucher * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10d38ceaf9SAlex Deucher * and/or sell copies of the Software, and to permit persons to whom the 11d38ceaf9SAlex Deucher * Software is furnished to do so, subject to the following conditions: 12d38ceaf9SAlex Deucher * 13d38ceaf9SAlex Deucher * The above copyright notice and this permission notice shall be included in 14d38ceaf9SAlex Deucher * all copies or substantial portions of the Software. 15d38ceaf9SAlex Deucher * 16d38ceaf9SAlex Deucher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17d38ceaf9SAlex Deucher * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18d38ceaf9SAlex Deucher * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19d38ceaf9SAlex Deucher * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20d38ceaf9SAlex Deucher * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21d38ceaf9SAlex Deucher * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22d38ceaf9SAlex Deucher * OTHER DEALINGS IN THE SOFTWARE. 23d38ceaf9SAlex Deucher * 24d38ceaf9SAlex Deucher * Authors: Dave Airlie 25d38ceaf9SAlex Deucher * Alex Deucher 26d38ceaf9SAlex Deucher * Jerome Glisse 27d38ceaf9SAlex Deucher */ 28d38ceaf9SAlex Deucher #ifndef __AMDGPU_OBJECT_H__ 29d38ceaf9SAlex Deucher #define __AMDGPU_OBJECT_H__ 30d38ceaf9SAlex Deucher 31d38ceaf9SAlex Deucher #include <drm/amdgpu_drm.h> 32d38ceaf9SAlex Deucher #include "amdgpu.h" 33d38ceaf9SAlex Deucher 349702d40dSChristian König #define AMDGPU_BO_INVALID_OFFSET LONG_MAX 35bf314ca3SChristian König #define AMDGPU_BO_MAX_PLACEMENTS 3 369702d40dSChristian König 37a906dbb1SChunming Zhou struct amdgpu_bo_param { 38a906dbb1SChunming Zhou unsigned long size; 39a906dbb1SChunming Zhou int byte_align; 40a906dbb1SChunming Zhou u32 domain; 41aa2b2e28SChunming Zhou u32 preferred_domain; 42a906dbb1SChunming Zhou u64 flags; 43a906dbb1SChunming Zhou enum ttm_bo_type type; 44a906dbb1SChunming Zhou struct reservation_object *resv; 45a906dbb1SChunming Zhou }; 46a906dbb1SChunming Zhou 47ec681545SChristian König /* bo virtual addresses in a vm */ 489124a398SChristian König struct amdgpu_bo_va_mapping { 49aebc5e6fSChristian König struct amdgpu_bo_va *bo_va; 509124a398SChristian König struct list_head list; 519124a398SChristian König struct rb_node rb; 529124a398SChristian König uint64_t start; 539124a398SChristian König uint64_t last; 549124a398SChristian König uint64_t __subtree_last; 559124a398SChristian König uint64_t offset; 569124a398SChristian König uint64_t flags; 579124a398SChristian König }; 589124a398SChristian König 59ec681545SChristian König /* User space allocated BO in a VM */ 609124a398SChristian König struct amdgpu_bo_va { 61ec681545SChristian König struct amdgpu_vm_bo_base base; 62ec681545SChristian König 639124a398SChristian König /* protected by bo being reserved */ 649124a398SChristian König unsigned ref_count; 659124a398SChristian König 6600b5cc83SChristian König /* all other members protected by the VM PD being reserved */ 6700b5cc83SChristian König struct dma_fence *last_pt_update; 6800b5cc83SChristian König 699124a398SChristian König /* mappings for this bo_va */ 709124a398SChristian König struct list_head invalids; 719124a398SChristian König struct list_head valids; 72cb7b6ec2SChristian König 73cb7b6ec2SChristian König /* If the mappings are cleared or filled */ 74cb7b6ec2SChristian König bool cleared; 759124a398SChristian König }; 769124a398SChristian König 779124a398SChristian König struct amdgpu_bo { 789124a398SChristian König /* Protected by tbo.reserved */ 796d7d9c5aSKent Russell u32 preferred_domains; 809124a398SChristian König u32 allowed_domains; 81bf314ca3SChristian König struct ttm_place placements[AMDGPU_BO_MAX_PLACEMENTS]; 829124a398SChristian König struct ttm_placement placement; 839124a398SChristian König struct ttm_buffer_object tbo; 849124a398SChristian König struct ttm_bo_kmap_obj kmap; 859124a398SChristian König u64 flags; 869124a398SChristian König unsigned pin_count; 879124a398SChristian König u64 tiling_flags; 889124a398SChristian König u64 metadata_flags; 899124a398SChristian König void *metadata; 909124a398SChristian König u32 metadata_size; 919124a398SChristian König unsigned prime_shared_count; 92*646b9025SChristian König /* per VM structure for page tables and with virtual addresses */ 93*646b9025SChristian König struct amdgpu_vm_bo_base *vm_bo; 949124a398SChristian König /* Constant after initialization */ 959124a398SChristian König struct drm_gem_object gem_base; 969124a398SChristian König struct amdgpu_bo *parent; 979124a398SChristian König struct amdgpu_bo *shadow; 989124a398SChristian König 999124a398SChristian König struct ttm_bo_kmap_obj dma_buf_vmap; 1009124a398SChristian König struct amdgpu_mn *mn; 101ed5b89c6SChristian König 102ed5b89c6SChristian König union { 1039124a398SChristian König struct list_head mn_list; 1049124a398SChristian König struct list_head shadow_list; 1059124a398SChristian König }; 106a46a2cd1SFelix Kuehling 107a46a2cd1SFelix Kuehling struct kgd_mem *kfd_bo; 108ed5b89c6SChristian König }; 1099124a398SChristian König 110b82485fdSAndres Rodriguez static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo) 111b82485fdSAndres Rodriguez { 112b82485fdSAndres Rodriguez return container_of(tbo, struct amdgpu_bo, tbo); 113b82485fdSAndres Rodriguez } 114b82485fdSAndres Rodriguez 115d38ceaf9SAlex Deucher /** 116d38ceaf9SAlex Deucher * amdgpu_mem_type_to_domain - return domain corresponding to mem_type 117d38ceaf9SAlex Deucher * @mem_type: ttm memory type 118d38ceaf9SAlex Deucher * 119d38ceaf9SAlex Deucher * Returns corresponding domain of the ttm mem_type 120d38ceaf9SAlex Deucher */ 121d38ceaf9SAlex Deucher static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type) 122d38ceaf9SAlex Deucher { 123d38ceaf9SAlex Deucher switch (mem_type) { 124d38ceaf9SAlex Deucher case TTM_PL_VRAM: 125d38ceaf9SAlex Deucher return AMDGPU_GEM_DOMAIN_VRAM; 126d38ceaf9SAlex Deucher case TTM_PL_TT: 127d38ceaf9SAlex Deucher return AMDGPU_GEM_DOMAIN_GTT; 128d38ceaf9SAlex Deucher case TTM_PL_SYSTEM: 129d38ceaf9SAlex Deucher return AMDGPU_GEM_DOMAIN_CPU; 130d38ceaf9SAlex Deucher case AMDGPU_PL_GDS: 131d38ceaf9SAlex Deucher return AMDGPU_GEM_DOMAIN_GDS; 132d38ceaf9SAlex Deucher case AMDGPU_PL_GWS: 133d38ceaf9SAlex Deucher return AMDGPU_GEM_DOMAIN_GWS; 134d38ceaf9SAlex Deucher case AMDGPU_PL_OA: 135d38ceaf9SAlex Deucher return AMDGPU_GEM_DOMAIN_OA; 136d38ceaf9SAlex Deucher default: 137d38ceaf9SAlex Deucher break; 138d38ceaf9SAlex Deucher } 139d38ceaf9SAlex Deucher return 0; 140d38ceaf9SAlex Deucher } 141d38ceaf9SAlex Deucher 142d38ceaf9SAlex Deucher /** 143d38ceaf9SAlex Deucher * amdgpu_bo_reserve - reserve bo 144d38ceaf9SAlex Deucher * @bo: bo structure 145d38ceaf9SAlex Deucher * @no_intr: don't return -ERESTARTSYS on pending signal 146d38ceaf9SAlex Deucher * 147d38ceaf9SAlex Deucher * Returns: 148d38ceaf9SAlex Deucher * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by 149d38ceaf9SAlex Deucher * a signal. Release all buffer reservations and return to user-space. 150d38ceaf9SAlex Deucher */ 151d38ceaf9SAlex Deucher static inline int amdgpu_bo_reserve(struct amdgpu_bo *bo, bool no_intr) 152d38ceaf9SAlex Deucher { 153a7d64de6SChristian König struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 154d38ceaf9SAlex Deucher int r; 155d38ceaf9SAlex Deucher 156dfd5e50eSChristian König r = ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL); 157d38ceaf9SAlex Deucher if (unlikely(r != 0)) { 158d38ceaf9SAlex Deucher if (r != -ERESTARTSYS) 159a7d64de6SChristian König dev_err(adev->dev, "%p reserve failed\n", bo); 160d38ceaf9SAlex Deucher return r; 161d38ceaf9SAlex Deucher } 162d38ceaf9SAlex Deucher return 0; 163d38ceaf9SAlex Deucher } 164d38ceaf9SAlex Deucher 165d38ceaf9SAlex Deucher static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo) 166d38ceaf9SAlex Deucher { 167d38ceaf9SAlex Deucher ttm_bo_unreserve(&bo->tbo); 168d38ceaf9SAlex Deucher } 169d38ceaf9SAlex Deucher 170d38ceaf9SAlex Deucher static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo) 171d38ceaf9SAlex Deucher { 172d38ceaf9SAlex Deucher return bo->tbo.num_pages << PAGE_SHIFT; 173d38ceaf9SAlex Deucher } 174d38ceaf9SAlex Deucher 175d38ceaf9SAlex Deucher static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo *bo) 176d38ceaf9SAlex Deucher { 177d38ceaf9SAlex Deucher return (bo->tbo.num_pages << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE; 178d38ceaf9SAlex Deucher } 179d38ceaf9SAlex Deucher 180d38ceaf9SAlex Deucher static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo *bo) 181d38ceaf9SAlex Deucher { 182d38ceaf9SAlex Deucher return (bo->tbo.mem.page_alignment << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE; 183d38ceaf9SAlex Deucher } 184d38ceaf9SAlex Deucher 185d38ceaf9SAlex Deucher /** 186d38ceaf9SAlex Deucher * amdgpu_bo_mmap_offset - return mmap offset of bo 187d38ceaf9SAlex Deucher * @bo: amdgpu object for which we query the offset 188d38ceaf9SAlex Deucher * 189d38ceaf9SAlex Deucher * Returns mmap offset of the object. 190d38ceaf9SAlex Deucher */ 191d38ceaf9SAlex Deucher static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo) 192d38ceaf9SAlex Deucher { 193d38ceaf9SAlex Deucher return drm_vma_node_offset_addr(&bo->tbo.vma_node); 194d38ceaf9SAlex Deucher } 195d38ceaf9SAlex Deucher 196b99f3103SNicolai Hähnle /** 1975422a28fSChristian König * amdgpu_bo_in_cpu_visible_vram - check if BO is (partly) in visible VRAM 1985422a28fSChristian König */ 1995422a28fSChristian König static inline bool amdgpu_bo_in_cpu_visible_vram(struct amdgpu_bo *bo) 2005422a28fSChristian König { 2015422a28fSChristian König struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 2025422a28fSChristian König unsigned fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 2035422a28fSChristian König struct drm_mm_node *node = bo->tbo.mem.mm_node; 2045422a28fSChristian König unsigned long pages_left; 2055422a28fSChristian König 2065422a28fSChristian König if (bo->tbo.mem.mem_type != TTM_PL_VRAM) 2075422a28fSChristian König return false; 2085422a28fSChristian König 2095422a28fSChristian König for (pages_left = bo->tbo.mem.num_pages; pages_left; 2105422a28fSChristian König pages_left -= node->size, node++) 2115422a28fSChristian König if (node->start < fpfn) 2125422a28fSChristian König return true; 2135422a28fSChristian König 2145422a28fSChristian König return false; 2155422a28fSChristian König } 2165422a28fSChristian König 2175422a28fSChristian König /** 218177ae09bSAndres Rodriguez * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced 219177ae09bSAndres Rodriguez */ 220177ae09bSAndres Rodriguez static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo) 221177ae09bSAndres Rodriguez { 222177ae09bSAndres Rodriguez return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC; 223177ae09bSAndres Rodriguez } 224177ae09bSAndres Rodriguez 225c704ab18SChristian König bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); 226c704ab18SChristian König void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain); 227c704ab18SChristian König 2283216c6b7SChunming Zhou int amdgpu_bo_create(struct amdgpu_device *adev, 2293216c6b7SChunming Zhou struct amdgpu_bo_param *bp, 230d38ceaf9SAlex Deucher struct amdgpu_bo **bo_ptr); 2319d903cbdSChristian König int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 2329d903cbdSChristian König unsigned long size, int align, 2339d903cbdSChristian König u32 domain, struct amdgpu_bo **bo_ptr, 2349d903cbdSChristian König u64 *gpu_addr, void **cpu_addr); 2357c204889SChristian König int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 2367c204889SChristian König unsigned long size, int align, 2377c204889SChristian König u32 domain, struct amdgpu_bo **bo_ptr, 2387c204889SChristian König u64 *gpu_addr, void **cpu_addr); 239aa1d562eSJunwei Zhang void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 240aa1d562eSJunwei Zhang void **cpu_addr); 241d38ceaf9SAlex Deucher int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr); 242f5e1c740SChristian König void *amdgpu_bo_kptr(struct amdgpu_bo *bo); 243d38ceaf9SAlex Deucher void amdgpu_bo_kunmap(struct amdgpu_bo *bo); 244d38ceaf9SAlex Deucher struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo); 245d38ceaf9SAlex Deucher void amdgpu_bo_unref(struct amdgpu_bo **bo); 2467b7c6c81SJunwei Zhang int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain); 247d38ceaf9SAlex Deucher int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, 2487b7c6c81SJunwei Zhang u64 min_offset, u64 max_offset); 249d38ceaf9SAlex Deucher int amdgpu_bo_unpin(struct amdgpu_bo *bo); 250d38ceaf9SAlex Deucher int amdgpu_bo_evict_vram(struct amdgpu_device *adev); 251d38ceaf9SAlex Deucher int amdgpu_bo_init(struct amdgpu_device *adev); 2526f752ec2SAndrey Grodzovsky int amdgpu_bo_late_init(struct amdgpu_device *adev); 253d38ceaf9SAlex Deucher void amdgpu_bo_fini(struct amdgpu_device *adev); 254d38ceaf9SAlex Deucher int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, 255d38ceaf9SAlex Deucher struct vm_area_struct *vma); 256d38ceaf9SAlex Deucher int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags); 257d38ceaf9SAlex Deucher void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags); 258d38ceaf9SAlex Deucher int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, 259d38ceaf9SAlex Deucher uint32_t metadata_size, uint64_t flags); 260d38ceaf9SAlex Deucher int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 261d38ceaf9SAlex Deucher size_t buffer_size, uint32_t *metadata_size, 262d38ceaf9SAlex Deucher uint64_t *flags); 263d38ceaf9SAlex Deucher void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 26466257db7SNicolai Hähnle bool evict, 265d38ceaf9SAlex Deucher struct ttm_mem_reg *new_mem); 266d38ceaf9SAlex Deucher int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); 267f54d1867SChris Wilson void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 268d38ceaf9SAlex Deucher bool shared); 269cdb7e8f2SChristian König u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo); 27020f4eff1SChunming Zhou int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, 27120f4eff1SChunming Zhou struct amdgpu_ring *ring, 27220f4eff1SChunming Zhou struct amdgpu_bo *bo, 27320f4eff1SChunming Zhou struct reservation_object *resv, 274f54d1867SChris Wilson struct dma_fence **fence, bool direct); 27582521316SRoger.He int amdgpu_bo_validate(struct amdgpu_bo *bo); 27620f4eff1SChunming Zhou int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, 27720f4eff1SChunming Zhou struct amdgpu_ring *ring, 27820f4eff1SChunming Zhou struct amdgpu_bo *bo, 27920f4eff1SChunming Zhou struct reservation_object *resv, 280f54d1867SChris Wilson struct dma_fence **fence, 28120f4eff1SChunming Zhou bool direct); 28284b74608SDeepak Sharma uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, 28384b74608SDeepak Sharma uint32_t domain); 284d38ceaf9SAlex Deucher 285d38ceaf9SAlex Deucher /* 286d38ceaf9SAlex Deucher * sub allocation 287d38ceaf9SAlex Deucher */ 288d38ceaf9SAlex Deucher 289d38ceaf9SAlex Deucher static inline uint64_t amdgpu_sa_bo_gpu_addr(struct amdgpu_sa_bo *sa_bo) 290d38ceaf9SAlex Deucher { 291d38ceaf9SAlex Deucher return sa_bo->manager->gpu_addr + sa_bo->soffset; 292d38ceaf9SAlex Deucher } 293d38ceaf9SAlex Deucher 294d38ceaf9SAlex Deucher static inline void * amdgpu_sa_bo_cpu_addr(struct amdgpu_sa_bo *sa_bo) 295d38ceaf9SAlex Deucher { 296d38ceaf9SAlex Deucher return sa_bo->manager->cpu_ptr + sa_bo->soffset; 297d38ceaf9SAlex Deucher } 298d38ceaf9SAlex Deucher 299d38ceaf9SAlex Deucher int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev, 300d38ceaf9SAlex Deucher struct amdgpu_sa_manager *sa_manager, 301d38ceaf9SAlex Deucher unsigned size, u32 align, u32 domain); 302d38ceaf9SAlex Deucher void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev, 303d38ceaf9SAlex Deucher struct amdgpu_sa_manager *sa_manager); 304d38ceaf9SAlex Deucher int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev, 305d38ceaf9SAlex Deucher struct amdgpu_sa_manager *sa_manager); 306bbf0b345SJunwei Zhang int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager, 307d38ceaf9SAlex Deucher struct amdgpu_sa_bo **sa_bo, 308d38ceaf9SAlex Deucher unsigned size, unsigned align); 309d38ceaf9SAlex Deucher void amdgpu_sa_bo_free(struct amdgpu_device *adev, 310d38ceaf9SAlex Deucher struct amdgpu_sa_bo **sa_bo, 311f54d1867SChris Wilson struct dma_fence *fence); 312d38ceaf9SAlex Deucher #if defined(CONFIG_DEBUG_FS) 313d38ceaf9SAlex Deucher void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, 314d38ceaf9SAlex Deucher struct seq_file *m); 315d38ceaf9SAlex Deucher #endif 316d38ceaf9SAlex Deucher 317d38ceaf9SAlex Deucher 318d38ceaf9SAlex Deucher #endif 319