1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ 2 /* 3 * Copyright 2020 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #ifndef _XE_RES_CURSOR_H_ 25 #define _XE_RES_CURSOR_H_ 26 27 #include <linux/scatterlist.h> 28 29 #include <drm/drm_pagemap.h> 30 #include <drm/ttm/ttm_placement.h> 31 #include <drm/ttm/ttm_range_manager.h> 32 #include <drm/ttm/ttm_resource.h> 33 #include <drm/ttm/ttm_tt.h> 34 35 #include "xe_bo.h" 36 #include "xe_device.h" 37 #include "xe_macros.h" 38 #include "xe_svm.h" 39 #include "xe_ttm_vram_mgr.h" 40 41 /** 42 * struct xe_res_cursor - state for walking over dma mapping, vram_mgr, 43 * stolen_mgr, and gtt_mgr allocations 44 */ 45 struct xe_res_cursor { 46 /** @start: Start of cursor */ 47 u64 start; 48 /** @size: Size of the current segment. */ 49 u64 size; 50 /** @remaining: Remaining bytes in cursor */ 51 u64 remaining; 52 /** @node: Opaque point current node cursor */ 53 void *node; 54 /** @mem_type: Memory type */ 55 u32 mem_type; 56 /** @sgl: Scatterlist for cursor */ 57 struct scatterlist *sgl; 58 /** @dma_addr: Current element in a struct drm_pagemap_addr array */ 59 const struct drm_pagemap_addr *dma_addr; 60 /** @mm: Buddy allocator for VRAM cursor */ 61 struct gpu_buddy *mm; 62 /** 63 * @dma_start: DMA start address for the current segment. 64 * This may be different to @dma_addr.addr since elements in 65 * the array may be coalesced to a single segment. 66 */ 67 u64 dma_start; 68 /** @dma_seg_size: Size of the current DMA segment. */ 69 u64 dma_seg_size; 70 }; 71 72 static struct gpu_buddy *xe_res_get_buddy(struct ttm_resource *res) 73 { 74 struct ttm_resource_manager *mgr; 75 76 mgr = ttm_manager_type(res->bo->bdev, res->mem_type); 77 return &to_xe_ttm_vram_mgr(mgr)->mm; 78 } 79 80 /** 81 * xe_res_first - initialize a xe_res_cursor 82 * 83 * @res: TTM resource object to walk 84 * @start: Start of the range 85 * @size: Size of the range 86 * @cur: cursor object to initialize 87 * 88 * Start walking over the range of allocations between @start and @size. 89 */ 90 static inline void xe_res_first(struct ttm_resource *res, 91 u64 start, u64 size, 92 struct xe_res_cursor *cur) 93 { 94 cur->sgl = NULL; 95 cur->dma_addr = NULL; 96 if (!res) 97 goto fallback; 98 99 XE_WARN_ON(start + size > res->size); 100 101 cur->mem_type = res->mem_type; 102 103 switch (cur->mem_type) { 104 case XE_PL_STOLEN: { 105 /* res->start is in pages (ttm_range_manager). */ 106 cur->start = (res->start << PAGE_SHIFT) + start; 107 cur->size = size; 108 cur->remaining = size; 109 cur->node = NULL; 110 cur->mm = NULL; 111 break; 112 } 113 case XE_PL_VRAM0: 114 case XE_PL_VRAM1: { 115 struct gpu_buddy_block *block; 116 struct list_head *head, *next; 117 struct gpu_buddy *mm = xe_res_get_buddy(res); 118 119 head = &to_xe_ttm_vram_mgr_resource(res)->blocks; 120 121 block = list_first_entry_or_null(head, 122 struct gpu_buddy_block, 123 link); 124 if (!block) 125 goto fallback; 126 127 while (start >= gpu_buddy_block_size(mm, block)) { 128 start -= gpu_buddy_block_size(mm, block); 129 130 next = block->link.next; 131 if (next != head) 132 block = list_entry(next, struct gpu_buddy_block, 133 link); 134 } 135 136 cur->mm = mm; 137 cur->start = gpu_buddy_block_offset(block) + start; 138 cur->size = min(gpu_buddy_block_size(mm, block) - start, 139 size); 140 cur->remaining = size; 141 cur->node = block; 142 break; 143 } 144 default: 145 goto fallback; 146 } 147 148 return; 149 150 fallback: 151 cur->start = start; 152 cur->size = size; 153 cur->remaining = size; 154 cur->node = NULL; 155 cur->mem_type = XE_PL_TT; 156 XE_WARN_ON(res && start + size > res->size); 157 } 158 159 static inline void __xe_res_sg_next(struct xe_res_cursor *cur) 160 { 161 struct scatterlist *sgl = cur->sgl; 162 u64 start = cur->start; 163 164 while (start >= sg_dma_len(sgl)) { 165 start -= sg_dma_len(sgl); 166 sgl = sg_next(sgl); 167 XE_WARN_ON(!sgl); 168 } 169 170 cur->start = start; 171 cur->size = sg_dma_len(sgl) - start; 172 cur->sgl = sgl; 173 } 174 175 /** 176 * __xe_res_dma_next() - Advance the cursor when end-of-segment is reached 177 * @cur: The cursor 178 */ 179 static inline void __xe_res_dma_next(struct xe_res_cursor *cur) 180 { 181 const struct drm_pagemap_addr *addr = cur->dma_addr; 182 u64 start = cur->start; 183 184 while (start >= cur->dma_seg_size) { 185 start -= cur->dma_seg_size; 186 addr++; 187 cur->dma_seg_size = PAGE_SIZE << addr->order; 188 } 189 cur->dma_start = addr->addr; 190 191 /* Coalesce array_elements */ 192 while (cur->dma_seg_size - start < cur->remaining) { 193 if (cur->dma_start + cur->dma_seg_size != addr[1].addr || 194 addr->proto != addr[1].proto) 195 break; 196 addr++; 197 cur->dma_seg_size += PAGE_SIZE << addr->order; 198 } 199 200 cur->dma_addr = addr; 201 cur->start = start; 202 cur->size = cur->dma_seg_size - start; 203 } 204 205 /** 206 * xe_res_first_sg - initialize a xe_res_cursor with a scatter gather table 207 * 208 * @sg: scatter gather table to walk 209 * @start: Start of the range 210 * @size: Size of the range 211 * @cur: cursor object to initialize 212 * 213 * Start walking over the range of allocations between @start and @size. 214 */ 215 static inline void xe_res_first_sg(const struct sg_table *sg, 216 u64 start, u64 size, 217 struct xe_res_cursor *cur) 218 { 219 XE_WARN_ON(!sg); 220 cur->node = NULL; 221 cur->start = start; 222 cur->remaining = size; 223 cur->size = 0; 224 cur->dma_addr = NULL; 225 cur->sgl = sg->sgl; 226 cur->mem_type = XE_PL_TT; 227 __xe_res_sg_next(cur); 228 } 229 230 /** 231 * xe_res_first_dma - initialize a xe_res_cursor with dma_addr array 232 * 233 * @dma_addr: struct drm_pagemap_addr array to walk 234 * @start: Start of the range 235 * @size: Size of the range 236 * @cur: cursor object to initialize 237 * 238 * Start walking over the range of allocations between @start and @size. 239 */ 240 static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr, 241 u64 start, u64 size, 242 struct xe_res_cursor *cur) 243 { 244 XE_WARN_ON(!dma_addr); 245 XE_WARN_ON(!IS_ALIGNED(start, PAGE_SIZE) || 246 !IS_ALIGNED(size, PAGE_SIZE)); 247 248 cur->node = NULL; 249 cur->start = start; 250 cur->remaining = size; 251 cur->dma_seg_size = PAGE_SIZE << dma_addr->order; 252 cur->dma_start = 0; 253 cur->size = 0; 254 cur->dma_addr = dma_addr; 255 __xe_res_dma_next(cur); 256 cur->sgl = NULL; 257 cur->mem_type = XE_PL_TT; 258 } 259 260 /** 261 * xe_res_next - advance the cursor 262 * 263 * @cur: the cursor to advance 264 * @size: number of bytes to move forward 265 * 266 * Move the cursor @size bytes forwrad, walking to the next node if necessary. 267 */ 268 static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) 269 { 270 struct gpu_buddy_block *block; 271 struct list_head *next; 272 u64 start; 273 274 XE_WARN_ON(size > cur->remaining); 275 276 cur->remaining -= size; 277 if (!cur->remaining) 278 return; 279 280 if (cur->size > size) { 281 cur->size -= size; 282 cur->start += size; 283 return; 284 } 285 286 if (cur->dma_addr) { 287 cur->start += size; 288 __xe_res_dma_next(cur); 289 return; 290 } 291 292 if (cur->sgl) { 293 cur->start += size; 294 __xe_res_sg_next(cur); 295 return; 296 } 297 298 switch (cur->mem_type) { 299 case XE_PL_STOLEN: 300 /* Just advance within the contiguous region. */ 301 cur->start += size; 302 cur->size = cur->remaining; 303 break; 304 case XE_PL_VRAM0: 305 case XE_PL_VRAM1: 306 start = size - cur->size; 307 block = cur->node; 308 309 next = block->link.next; 310 block = list_entry(next, struct gpu_buddy_block, link); 311 312 313 while (start >= gpu_buddy_block_size(cur->mm, block)) { 314 start -= gpu_buddy_block_size(cur->mm, block); 315 316 next = block->link.next; 317 block = list_entry(next, struct gpu_buddy_block, link); 318 } 319 320 cur->start = gpu_buddy_block_offset(block) + start; 321 cur->size = min(gpu_buddy_block_size(cur->mm, block) - start, 322 cur->remaining); 323 cur->node = block; 324 break; 325 default: 326 return; 327 } 328 } 329 330 /** 331 * xe_res_dma - return dma address of cursor at current position 332 * 333 * @cur: the cursor to return the dma address from 334 */ 335 static inline u64 xe_res_dma(const struct xe_res_cursor *cur) 336 { 337 if (cur->dma_addr) 338 return cur->dma_start + cur->start; 339 else if (cur->sgl) 340 return sg_dma_address(cur->sgl) + cur->start; 341 else 342 return cur->start; 343 } 344 345 /** 346 * xe_res_is_vram() - Whether the cursor current dma address points to 347 * same-device VRAM 348 * @cur: The cursor. 349 * 350 * Return: true iff the address returned by xe_res_dma() points to internal vram. 351 */ 352 static inline bool xe_res_is_vram(const struct xe_res_cursor *cur) 353 { 354 if (cur->dma_addr) 355 return cur->dma_addr->proto == XE_INTERCONNECT_VRAM; 356 357 switch (cur->mem_type) { 358 case XE_PL_STOLEN: 359 case XE_PL_VRAM0: 360 case XE_PL_VRAM1: 361 return true; 362 default: 363 break; 364 } 365 366 return false; 367 } 368 #endif 369