1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright 2020-2021 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #include <linux/types.h>
24 #include <linux/dma-direction.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/migrate.h>
27 #include "amdgpu_sync.h"
28 #include "amdgpu_object.h"
29 #include "amdgpu_vm.h"
30 #include "amdgpu_res_cursor.h"
31 #include "kfd_priv.h"
32 #include "kfd_svm.h"
33 #include "kfd_migrate.h"
34 #include "kfd_smi_events.h"
35
36 #ifdef dev_fmt
37 #undef dev_fmt
38 #endif
39 #define dev_fmt(fmt) "kfd_migrate: " fmt
40
41 static u64
svm_migrate_direct_mapping_addr(struct amdgpu_device * adev,u64 addr)42 svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, u64 addr)
43 {
44 return addr + amdgpu_ttm_domain_start(adev, TTM_PL_VRAM);
45 }
46
47 static int
svm_migrate_gart_map(struct amdgpu_ring * ring,struct amdgpu_ttm_buffer_entity * entity,u64 npages,dma_addr_t * addr,u64 * gart_addr,u64 flags)48 svm_migrate_gart_map(struct amdgpu_ring *ring,
49 struct amdgpu_ttm_buffer_entity *entity,
50 u64 npages,
51 dma_addr_t *addr, u64 *gart_addr, u64 flags)
52 {
53 struct amdgpu_device *adev = ring->adev;
54 struct amdgpu_job *job;
55 unsigned int num_dw, num_bytes;
56 struct dma_fence *fence;
57 u64 src_addr, dst_addr;
58 u64 pte_flags;
59 void *cpu_addr;
60 int r;
61
62 *gart_addr = amdgpu_compute_gart_address(&adev->gmc, entity, 0);
63
64 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
65 num_bytes = npages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
66
67 r = amdgpu_job_alloc_with_ib(adev, &entity->base,
68 AMDGPU_FENCE_OWNER_UNDEFINED,
69 num_dw * 4 + num_bytes,
70 AMDGPU_IB_POOL_DELAYED,
71 AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP,
72 &job);
73 if (r)
74 return r;
75
76 src_addr = num_dw * 4;
77 src_addr += job->ibs[0].gpu_addr;
78
79 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
80 dst_addr += (entity->gart_window_offs[0] >> AMDGPU_GPU_PAGE_SHIFT) * 8;
81 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
82 dst_addr, num_bytes, 0);
83
84 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
85 WARN_ON(job->ibs[0].length_dw > num_dw);
86
87 pte_flags = AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE;
88 pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED;
89 if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
90 pte_flags |= AMDGPU_PTE_WRITEABLE;
91 pte_flags |= adev->gart.gart_pte_flags;
92
93 cpu_addr = &job->ibs[0].ptr[num_dw];
94
95 amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
96 fence = amdgpu_job_submit(job);
97 dma_fence_put(fence);
98
99 return r;
100 }
101
102 /**
103 * svm_migrate_copy_memory_gart - sdma copy data between ram and vram
104 *
105 * @adev: amdgpu device the sdma ring running
106 * @sys: system DMA pointer to be copied
107 * @vram: vram destination DMA pointer
108 * @npages: number of pages to copy
109 * @direction: enum MIGRATION_COPY_DIR
110 * @mfence: output, sdma fence to signal after sdma is done
111 *
112 * ram address uses GART table continuous entries mapping to ram pages,
113 * vram address uses direct mapping of vram pages, which must have npages
114 * number of continuous pages.
115 * GART update and sdma uses same buf copy function ring, sdma is splited to
116 * multiple GTT_MAX_PAGES transfer, all sdma operations are serialized, wait for
117 * the last sdma finish fence which is returned to check copy memory is done.
118 *
119 * Context: Process context
120 *
121 * Return:
122 * 0 - OK, otherwise error code
123 */
124
125 static int
svm_migrate_copy_memory_gart(struct amdgpu_device * adev,dma_addr_t * sys,u64 * vram,u64 npages,enum MIGRATION_COPY_DIR direction,struct dma_fence ** mfence)126 svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
127 u64 *vram, u64 npages,
128 enum MIGRATION_COPY_DIR direction,
129 struct dma_fence **mfence)
130 {
131 const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT);
132 struct amdgpu_ring *ring;
133 struct amdgpu_ttm_buffer_entity *entity;
134 u64 gart_s, gart_d;
135 struct dma_fence *next;
136 u64 size;
137 int r = 0;
138
139 ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
140 entity = &adev->mman.move_entities[0];
141
142 mutex_lock(&entity->lock);
143
144 while (npages) {
145 size = min(GTT_MAX_PAGES, npages);
146
147 if (direction == FROM_VRAM_TO_RAM) {
148 gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
149 r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0);
150
151 } else if (direction == FROM_RAM_TO_VRAM) {
152 r = svm_migrate_gart_map(ring, entity, size, sys, &gart_s,
153 KFD_IOCTL_SVM_FLAG_GPU_RO);
154 gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
155 }
156 if (r) {
157 dev_err(adev->dev, "fail %d create gart mapping\n", r);
158 goto out_unlock;
159 }
160
161 r = amdgpu_copy_buffer(adev, entity,
162 gart_s, gart_d, size * PAGE_SIZE,
163 NULL, &next, true, 0);
164 if (r) {
165 dev_err(adev->dev, "fail %d to copy memory\n", r);
166 goto out_unlock;
167 }
168
169 dma_fence_put(*mfence);
170 *mfence = next;
171 npages -= size;
172 if (npages) {
173 sys += size;
174 vram += size;
175 }
176 }
177
178 out_unlock:
179 mutex_unlock(&entity->lock);
180
181 return r;
182 }
183
184 /**
185 * svm_migrate_copy_done - wait for memory copy sdma is done
186 *
187 * @adev: amdgpu device the sdma memory copy is executing on
188 * @mfence: migrate fence
189 *
190 * Wait for dma fence is signaled, if the copy ssplit into multiple sdma
191 * operations, this is the last sdma operation fence.
192 *
193 * Context: called after svm_migrate_copy_memory
194 *
195 * Return:
196 * 0 - success
197 * otherwise - error code from dma fence signal
198 */
199 static int
svm_migrate_copy_done(struct amdgpu_device * adev,struct dma_fence * mfence)200 svm_migrate_copy_done(struct amdgpu_device *adev, struct dma_fence *mfence)
201 {
202 int r = 0;
203
204 if (mfence) {
205 r = dma_fence_wait(mfence, false);
206 dma_fence_put(mfence);
207 pr_debug("sdma copy memory fence done\n");
208 }
209
210 return r;
211 }
212
213 unsigned long
svm_migrate_addr_to_pfn(struct amdgpu_device * adev,unsigned long addr)214 svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
215 {
216 return (addr + adev->kfd.pgmap.range.start) >> PAGE_SHIFT;
217 }
218
219 static void
svm_migrate_get_vram_page(struct svm_range * prange,unsigned long pfn)220 svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
221 {
222 struct page *page;
223
224 page = pfn_to_page(pfn);
225 svm_range_bo_ref(prange->svm_bo);
226 page->zone_device_data = prange->svm_bo;
227 zone_device_page_init(page, page_pgmap(page), 0);
228 }
229
230 static void
svm_migrate_put_vram_page(struct amdgpu_device * adev,unsigned long addr)231 svm_migrate_put_vram_page(struct amdgpu_device *adev, unsigned long addr)
232 {
233 struct page *page;
234
235 page = pfn_to_page(svm_migrate_addr_to_pfn(adev, addr));
236 unlock_page(page);
237 put_page(page);
238 }
239
240 static unsigned long
svm_migrate_addr(struct amdgpu_device * adev,struct page * page)241 svm_migrate_addr(struct amdgpu_device *adev, struct page *page)
242 {
243 unsigned long addr;
244
245 addr = page_to_pfn(page) << PAGE_SHIFT;
246 return (addr - adev->kfd.pgmap.range.start);
247 }
248
249 static struct page *
svm_migrate_get_sys_page(struct vm_area_struct * vma,unsigned long addr)250 svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
251 {
252 struct page *page;
253
254 page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
255 if (page)
256 lock_page(page);
257
258 return page;
259 }
260
svm_migrate_successful_pages(struct migrate_vma * migrate)261 static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
262 {
263 unsigned long mpages = 0;
264 unsigned long i;
265
266 for (i = 0; i < migrate->npages; i++) {
267 if (migrate->dst[i] & MIGRATE_PFN_VALID &&
268 migrate->src[i] & MIGRATE_PFN_MIGRATE)
269 mpages++;
270 }
271 return mpages;
272 }
273
274 static int
svm_migrate_copy_to_vram(struct kfd_node * node,struct svm_range * prange,struct migrate_vma * migrate,struct dma_fence ** mfence,dma_addr_t * scratch,u64 ttm_res_offset)275 svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
276 struct migrate_vma *migrate, struct dma_fence **mfence,
277 dma_addr_t *scratch, u64 ttm_res_offset)
278 {
279 u64 npages = migrate->npages;
280 struct amdgpu_device *adev = node->adev;
281 struct device *dev = adev->dev;
282 struct amdgpu_res_cursor cursor;
283 u64 mpages = 0;
284 dma_addr_t *src;
285 u64 *dst;
286 u64 i, j;
287 int r = 0;
288
289 pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
290 prange->last, ttm_res_offset);
291
292 src = scratch;
293 dst = (u64 *)(scratch + npages);
294
295 amdgpu_res_first(prange->ttm_res, ttm_res_offset,
296 npages << PAGE_SHIFT, &cursor);
297 for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {
298 struct page *spage;
299
300 if (migrate->src[i] & MIGRATE_PFN_MIGRATE) {
301 dst[i] = cursor.start + (j << PAGE_SHIFT);
302 migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
303 svm_migrate_get_vram_page(prange, migrate->dst[i]);
304 migrate->dst[i] = migrate_pfn(migrate->dst[i]);
305 mpages++;
306 }
307 spage = migrate_pfn_to_page(migrate->src[i]);
308 if (spage && !is_zone_device_page(spage)) {
309 src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
310 DMA_BIDIRECTIONAL);
311 r = dma_mapping_error(dev, src[i]);
312 if (r) {
313 src[i] = 0;
314 dev_err(dev, "%s: fail %d dma_map_page\n",
315 __func__, r);
316 goto out_free_vram_pages;
317 }
318 } else {
319 if (j) {
320 r = svm_migrate_copy_memory_gart(
321 adev, src + i - j,
322 dst + i - j, j,
323 FROM_RAM_TO_VRAM,
324 mfence);
325 if (r)
326 goto out_free_vram_pages;
327 amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
328 j = 0;
329 } else {
330 amdgpu_res_next(&cursor, PAGE_SIZE);
331 }
332 continue;
333 }
334
335 pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
336 src[i] >> PAGE_SHIFT, page_to_pfn(spage));
337
338 /* accumulated j + 1 pages reach end of current drm_buddy_block */
339 if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
340 r = svm_migrate_copy_memory_gart(adev, src + i - j,
341 dst + i - j, j + 1,
342 FROM_RAM_TO_VRAM,
343 mfence);
344 if (r)
345 goto out_free_vram_pages;
346 amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
347 j = 0;
348 } else {
349 j++;
350 }
351 }
352
353 if (j > 0)
354 r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
355 FROM_RAM_TO_VRAM, mfence);
356
357 out_free_vram_pages:
358 if (r) {
359 pr_debug("failed %d to copy memory to vram\n", r);
360 for (i = 0; i < npages && mpages; i++) {
361 if (!dst[i])
362 continue;
363 svm_migrate_put_vram_page(adev, dst[i]);
364 migrate->dst[i] = 0;
365 mpages--;
366 }
367 }
368
369 #ifdef DEBUG_FORCE_MIXED_DOMAINS
370 for (i = 0, j = 0; i < npages; i += 4, j++) {
371 if (j & 1)
372 continue;
373 svm_migrate_put_vram_page(adev, dst[i]);
374 migrate->dst[i] = 0;
375 svm_migrate_put_vram_page(adev, dst[i + 1]);
376 migrate->dst[i + 1] = 0;
377 svm_migrate_put_vram_page(adev, dst[i + 2]);
378 migrate->dst[i + 2] = 0;
379 svm_migrate_put_vram_page(adev, dst[i + 3]);
380 migrate->dst[i + 3] = 0;
381 }
382 #endif
383
384 return r;
385 }
386
387 static long
svm_migrate_vma_to_vram(struct kfd_node * node,struct svm_range * prange,struct vm_area_struct * vma,u64 start,u64 end,uint32_t trigger,u64 ttm_res_offset)388 svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
389 struct vm_area_struct *vma, u64 start,
390 u64 end, uint32_t trigger, u64 ttm_res_offset)
391 {
392 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
393 u64 npages = (end - start) >> PAGE_SHIFT;
394 struct amdgpu_device *adev = node->adev;
395 struct kfd_process_device *pdd;
396 struct dma_fence *mfence = NULL;
397 struct migrate_vma migrate = { 0 };
398 unsigned long cpages = 0;
399 unsigned long mpages = 0;
400 dma_addr_t *scratch;
401 void *buf;
402 int r = -ENOMEM;
403
404 memset(&migrate, 0, sizeof(migrate));
405 migrate.vma = vma;
406 migrate.start = start;
407 migrate.end = end;
408 migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
409 migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
410
411 buf = kvcalloc(npages,
412 2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
413 GFP_KERNEL);
414 if (!buf)
415 goto out;
416
417 migrate.src = buf;
418 migrate.dst = migrate.src + npages;
419 scratch = (dma_addr_t *)(migrate.dst + npages);
420
421 kfd_smi_event_migration_start(node, p->lead_thread,
422 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
423 0, node->id, prange->prefetch_loc,
424 prange->preferred_loc, trigger);
425
426 r = migrate_vma_setup(&migrate);
427 if (r) {
428 dev_err(adev->dev, "%s: vma setup fail %d range [0x%lx 0x%lx]\n",
429 __func__, r, prange->start, prange->last);
430 goto out_free;
431 }
432
433 cpages = migrate.cpages;
434 if (!cpages) {
435 pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",
436 prange->start, prange->last);
437 goto out_free;
438 }
439 if (cpages != npages)
440 pr_debug("partial migration, 0x%lx/0x%llx pages collected\n",
441 cpages, npages);
442 else
443 pr_debug("0x%lx pages collected\n", cpages);
444
445 r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);
446 migrate_vma_pages(&migrate);
447
448 svm_migrate_copy_done(adev, mfence);
449 migrate_vma_finalize(&migrate);
450
451 mpages = svm_migrate_successful_pages(&migrate);
452 pr_debug("migrated/collected/requested 0x%lx/0x%lx/0x%lx\n",
453 mpages, cpages, migrate.npages);
454
455 svm_range_dma_unmap_dev(adev->dev, scratch, 0, npages);
456
457 out_free:
458 kvfree(buf);
459 kfd_smi_event_migration_end(node, p->lead_thread,
460 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
461 0, node->id, trigger, r);
462 out:
463 if (!r && mpages) {
464 pdd = svm_range_get_pdd_by_node(prange, node);
465 if (pdd)
466 WRITE_ONCE(pdd->page_in, pdd->page_in + mpages);
467
468 return mpages;
469 }
470 return r;
471 }
472
473 /**
474 * svm_migrate_ram_to_vram - migrate svm range from system to device
475 * @prange: range structure
476 * @best_loc: the device to migrate to
477 * @start_mgr: start page to migrate
478 * @last_mgr: last page to migrate
479 * @mm: the process mm structure
480 * @trigger: reason of migration
481 *
482 * Context: Process context, caller hold mmap read lock, svms lock, prange lock
483 *
484 * Return:
485 * 0 - OK, otherwise error code
486 */
487 static int
svm_migrate_ram_to_vram(struct svm_range * prange,uint32_t best_loc,unsigned long start_mgr,unsigned long last_mgr,struct mm_struct * mm,uint32_t trigger)488 svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
489 unsigned long start_mgr, unsigned long last_mgr,
490 struct mm_struct *mm, uint32_t trigger)
491 {
492 unsigned long addr, start, end;
493 struct vm_area_struct *vma;
494 u64 ttm_res_offset;
495 struct kfd_node *node;
496 unsigned long mpages = 0;
497 long r = 0;
498
499 if (start_mgr < prange->start || last_mgr > prange->last) {
500 pr_debug("range [0x%lx 0x%lx] out prange [0x%lx 0x%lx]\n",
501 start_mgr, last_mgr, prange->start, prange->last);
502 return -EFAULT;
503 }
504
505 node = svm_range_get_node_by_id(prange, best_loc);
506 if (!node) {
507 pr_debug("failed to get kfd node by id 0x%x\n", best_loc);
508 return -ENODEV;
509 }
510
511 pr_debug("svms 0x%p [0x%lx 0x%lx] in [0x%lx 0x%lx] to gpu 0x%x\n",
512 prange->svms, start_mgr, last_mgr, prange->start, prange->last,
513 best_loc);
514
515 start = start_mgr << PAGE_SHIFT;
516 end = (last_mgr + 1) << PAGE_SHIFT;
517
518 r = amdgpu_amdkfd_reserve_mem_limit(node->adev,
519 prange->npages * PAGE_SIZE,
520 KFD_IOC_ALLOC_MEM_FLAGS_VRAM,
521 node->xcp ? node->xcp->id : 0);
522 if (r) {
523 dev_dbg(node->adev->dev, "failed to reserve VRAM, r: %ld\n", r);
524 return -ENOSPC;
525 }
526
527 r = svm_range_vram_node_new(node, prange, true);
528 if (r) {
529 dev_dbg(node->adev->dev, "fail %ld to alloc vram\n", r);
530 goto out;
531 }
532 ttm_res_offset = (start_mgr - prange->start + prange->offset) << PAGE_SHIFT;
533
534 for (addr = start; addr < end;) {
535 unsigned long next;
536
537 vma = vma_lookup(mm, addr);
538 if (!vma)
539 break;
540
541 next = min(vma->vm_end, end);
542 r = svm_migrate_vma_to_vram(node, prange, vma, addr, next, trigger, ttm_res_offset);
543 if (r < 0) {
544 pr_debug("failed %ld to migrate\n", r);
545 break;
546 } else {
547 mpages += r;
548 }
549 ttm_res_offset += next - addr;
550 addr = next;
551 }
552
553 if (mpages) {
554 prange->actual_loc = best_loc;
555 prange->vram_pages += mpages;
556 } else if (!prange->actual_loc) {
557 /* if no page migrated and all pages from prange are at
558 * sys ram drop svm_bo got from svm_range_vram_node_new
559 */
560 svm_range_vram_node_free(prange);
561 }
562
563 out:
564 amdgpu_amdkfd_unreserve_mem_limit(node->adev,
565 prange->npages * PAGE_SIZE,
566 KFD_IOC_ALLOC_MEM_FLAGS_VRAM,
567 node->xcp ? node->xcp->id : 0);
568 return r < 0 ? r : 0;
569 }
570
svm_migrate_folio_free(struct folio * folio)571 static void svm_migrate_folio_free(struct folio *folio)
572 {
573 struct page *page = &folio->page;
574 struct svm_range_bo *svm_bo = page->zone_device_data;
575
576 if (svm_bo) {
577 pr_debug_ratelimited("ref: %d\n", kref_read(&svm_bo->kref));
578 svm_range_bo_unref_async(svm_bo);
579 }
580 }
581
582 static int
svm_migrate_copy_to_ram(struct amdgpu_device * adev,struct svm_range * prange,struct migrate_vma * migrate,struct dma_fence ** mfence,dma_addr_t * scratch,u64 npages)583 svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
584 struct migrate_vma *migrate, struct dma_fence **mfence,
585 dma_addr_t *scratch, u64 npages)
586 {
587 struct device *dev = adev->dev;
588 struct page *dpage = NULL;
589 dma_addr_t *dst;
590 u64 *src;
591
592 u64 i = 0, j;
593 u64 addr;
594 int r = 0;
595
596 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
597 prange->last);
598
599 addr = migrate->start;
600
601 src = (u64 *)(scratch + npages);
602 dst = scratch;
603
604 for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {
605 struct page *spage;
606
607 spage = migrate_pfn_to_page(migrate->src[i]);
608 if (!spage || !is_zone_device_page(spage)) {
609 pr_debug("invalid page. Could be in CPU already svms 0x%p [0x%lx 0x%lx]\n",
610 prange->svms, prange->start, prange->last);
611 if (j) {
612 r = svm_migrate_copy_memory_gart(adev, dst + i - j,
613 src + i - j, j,
614 FROM_VRAM_TO_RAM,
615 mfence);
616 if (r)
617 goto out_oom;
618 j = 0;
619 }
620 continue;
621 }
622 src[i] = svm_migrate_addr(adev, spage);
623 if (j > 0 && src[i] != src[i - 1] + PAGE_SIZE) {
624 r = svm_migrate_copy_memory_gart(adev, dst + i - j,
625 src + i - j, j,
626 FROM_VRAM_TO_RAM,
627 mfence);
628 if (r)
629 goto out_oom;
630 j = 0;
631 }
632
633 dpage = svm_migrate_get_sys_page(migrate->vma, addr);
634 if (!dpage) {
635 pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
636 prange->svms, prange->start, prange->last);
637 r = -ENOMEM;
638 goto out_oom;
639 }
640
641 dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
642 r = dma_mapping_error(dev, dst[i]);
643 if (r) {
644 dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
645 dst[i] = 0;
646 goto out_oom;
647 }
648
649 pr_debug_ratelimited("dma mapping dst to 0x%llx, pfn 0x%lx\n",
650 dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
651
652 migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
653
654 dpage = NULL;
655 j++;
656 }
657
658 if (j > 0)
659 r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
660 FROM_VRAM_TO_RAM, mfence);
661 out_oom:
662 if (r) {
663 pr_debug("failed %d copy to ram\n", r);
664
665 /* first release current dpage when dma_map_page fail */
666 if (dpage) {
667 unlock_page(dpage);
668 put_page(dpage);
669 }
670
671 /* release previous allocated sys pages and unmap dma address */
672 while (i--) {
673
674 if (dst[i]) {
675 dma_unmap_page(dev, dst[i], PAGE_SIZE,
676 DMA_BIDIRECTIONAL);
677 dst[i] = 0;
678 }
679
680 dpage = migrate_pfn_to_page(migrate->dst[i]);
681 if (!dpage)
682 continue;
683
684 unlock_page(dpage);
685 put_page(dpage);
686 migrate->dst[i] = 0;
687 }
688 }
689
690 return r;
691 }
692
693 /**
694 * svm_migrate_vma_to_ram - migrate range inside one vma from device to system
695 *
696 * @prange: svm range structure
697 * @vma: vm_area_struct that range [start, end] belongs to
698 * @start: range start virtual address in pages
699 * @end: range end virtual address in pages
700 * @node: kfd node device to migrate from
701 * @trigger: reason of migration
702 * @fault_page: is from vmf->page, svm_migrate_to_ram(), this is CPU page fault callback
703 *
704 * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
705 *
706 * Return:
707 * negative values - indicate error
708 * positive values or zero - number of pages got migrated
709 */
710 static long
svm_migrate_vma_to_ram(struct kfd_node * node,struct svm_range * prange,struct vm_area_struct * vma,u64 start,u64 end,uint32_t trigger,struct page * fault_page)711 svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
712 struct vm_area_struct *vma, u64 start, u64 end,
713 uint32_t trigger, struct page *fault_page)
714 {
715 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
716 u64 npages = (end - start) >> PAGE_SHIFT;
717 unsigned long cpages = 0;
718 unsigned long mpages = 0;
719 struct amdgpu_device *adev = node->adev;
720 struct kfd_process_device *pdd;
721 struct dma_fence *mfence = NULL;
722 struct migrate_vma migrate = { 0 };
723 dma_addr_t *scratch;
724 void *buf;
725 int r = -ENOMEM;
726
727 memset(&migrate, 0, sizeof(migrate));
728 migrate.vma = vma;
729 migrate.start = start;
730 migrate.end = end;
731 migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
732 if (adev->gmc.xgmi.connected_to_cpu)
733 migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;
734 else
735 migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
736
737 buf = kvcalloc(npages,
738 2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),
739 GFP_KERNEL);
740 if (!buf)
741 goto out;
742
743 migrate.src = buf;
744 migrate.dst = migrate.src + npages;
745 migrate.fault_page = fault_page;
746 scratch = (dma_addr_t *)(migrate.dst + npages);
747
748 kfd_smi_event_migration_start(node, p->lead_thread,
749 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
750 node->id, 0, prange->prefetch_loc,
751 prange->preferred_loc, trigger);
752
753 r = migrate_vma_setup(&migrate);
754 if (r) {
755 dev_err(adev->dev, "%s: vma setup fail %d range [0x%lx 0x%lx]\n",
756 __func__, r, prange->start, prange->last);
757 goto out_free;
758 }
759
760 cpages = migrate.cpages;
761 if (!cpages) {
762 pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",
763 prange->start, prange->last);
764 goto out_free;
765 }
766 if (cpages != npages)
767 pr_debug("partial migration, 0x%lx/0x%llx pages collected\n",
768 cpages, npages);
769 else
770 pr_debug("0x%lx pages collected\n", cpages);
771
772 r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
773 scratch, npages);
774 migrate_vma_pages(&migrate);
775
776 mpages = svm_migrate_successful_pages(&migrate);
777 pr_debug("migrated/collected/requested 0x%lx/0x%lx/0x%lx\n",
778 mpages, cpages, migrate.npages);
779
780 svm_migrate_copy_done(adev, mfence);
781 migrate_vma_finalize(&migrate);
782
783 svm_range_dma_unmap_dev(adev->dev, scratch, 0, npages);
784
785 out_free:
786 kvfree(buf);
787 kfd_smi_event_migration_end(node, p->lead_thread,
788 start >> PAGE_SHIFT, end >> PAGE_SHIFT,
789 node->id, 0, trigger, r);
790 out:
791 if (!r && mpages) {
792 pdd = svm_range_get_pdd_by_node(prange, node);
793 if (pdd)
794 WRITE_ONCE(pdd->page_out, pdd->page_out + mpages);
795 }
796
797 return r ? r : mpages;
798 }
799
800 /**
801 * svm_migrate_vram_to_ram - migrate svm range from device to system
802 * @prange: range structure
803 * @mm: process mm, use current->mm if NULL
804 * @start_mgr: start page need be migrated to sys ram
805 * @last_mgr: last page need be migrated to sys ram
806 * @trigger: reason of migration
807 * @fault_page: is from vmf->page, svm_migrate_to_ram(), this is CPU page fault callback
808 *
809 * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
810 *
811 * Return:
812 * 0 - OK, otherwise error code
813 */
svm_migrate_vram_to_ram(struct svm_range * prange,struct mm_struct * mm,unsigned long start_mgr,unsigned long last_mgr,uint32_t trigger,struct page * fault_page)814 int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
815 unsigned long start_mgr, unsigned long last_mgr,
816 uint32_t trigger, struct page *fault_page)
817 {
818 struct kfd_node *node;
819 struct vm_area_struct *vma;
820 unsigned long addr;
821 unsigned long start;
822 unsigned long end;
823 unsigned long mpages = 0;
824 long r = 0;
825
826 /* this pragne has no any vram page to migrate to sys ram */
827 if (!prange->actual_loc) {
828 pr_debug("[0x%lx 0x%lx] already migrated to ram\n",
829 prange->start, prange->last);
830 return 0;
831 }
832
833 if (start_mgr < prange->start || last_mgr > prange->last) {
834 pr_debug("range [0x%lx 0x%lx] out prange [0x%lx 0x%lx]\n",
835 start_mgr, last_mgr, prange->start, prange->last);
836 return -EFAULT;
837 }
838
839 node = svm_range_get_node_by_id(prange, prange->actual_loc);
840 if (!node) {
841 pr_debug("failed to get kfd node by id 0x%x\n", prange->actual_loc);
842 return -ENODEV;
843 }
844 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu 0x%x to ram\n",
845 prange->svms, prange, start_mgr, last_mgr,
846 prange->actual_loc);
847
848 start = start_mgr << PAGE_SHIFT;
849 end = (last_mgr + 1) << PAGE_SHIFT;
850
851 for (addr = start; addr < end;) {
852 unsigned long next;
853
854 vma = vma_lookup(mm, addr);
855 if (!vma) {
856 pr_debug("failed to find vma for prange %p\n", prange);
857 r = -EFAULT;
858 break;
859 }
860
861 next = min(vma->vm_end, end);
862 r = svm_migrate_vma_to_ram(node, prange, vma, addr, next, trigger,
863 fault_page);
864 if (r < 0) {
865 pr_debug("failed %ld to migrate prange %p\n", r, prange);
866 break;
867 } else {
868 mpages += r;
869 }
870 addr = next;
871 }
872
873 if (r >= 0) {
874 WARN_ONCE(prange->vram_pages < mpages,
875 "Recorded vram pages(0x%llx) should not be less than migration pages(0x%lx).",
876 prange->vram_pages, mpages);
877 prange->vram_pages -= mpages;
878
879 /* prange does not have vram page set its actual_loc to system
880 * and drop its svm_bo ref
881 */
882 if (prange->vram_pages == 0 && prange->ttm_res) {
883 prange->actual_loc = 0;
884 svm_range_vram_node_free(prange);
885 }
886 }
887
888 return r < 0 ? r : 0;
889 }
890
891 /**
892 * svm_migrate_vram_to_vram - migrate svm range from device to device
893 * @prange: range structure
894 * @best_loc: the device to migrate to
895 * @start: start page need be migrated to sys ram
896 * @last: last page need be migrated to sys ram
897 * @mm: process mm, use current->mm if NULL
898 * @trigger: reason of migration
899 *
900 * Context: Process context, caller hold mmap read lock, svms lock, prange lock
901 *
902 * migrate all vram pages in prange to sys ram, then migrate
903 * [start, last] pages from sys ram to gpu node best_loc.
904 *
905 * Return:
906 * 0 - OK, otherwise error code
907 */
908 static int
svm_migrate_vram_to_vram(struct svm_range * prange,uint32_t best_loc,unsigned long start,unsigned long last,struct mm_struct * mm,uint32_t trigger)909 svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
910 unsigned long start, unsigned long last,
911 struct mm_struct *mm, uint32_t trigger)
912 {
913 int r, retries = 3;
914
915 /*
916 * TODO: for both devices with PCIe large bar or on same xgmi hive, skip
917 * system memory as migration bridge
918 */
919
920 pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
921
922 do {
923 r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,
924 trigger, NULL);
925 if (r)
926 return r;
927 } while (prange->actual_loc && --retries);
928
929 if (prange->actual_loc)
930 return -EDEADLK;
931
932 return svm_migrate_ram_to_vram(prange, best_loc, start, last, mm, trigger);
933 }
934
935 int
svm_migrate_to_vram(struct svm_range * prange,uint32_t best_loc,unsigned long start,unsigned long last,struct mm_struct * mm,uint32_t trigger)936 svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
937 unsigned long start, unsigned long last,
938 struct mm_struct *mm, uint32_t trigger)
939 {
940 if (!prange->actual_loc || prange->actual_loc == best_loc)
941 return svm_migrate_ram_to_vram(prange, best_loc, start, last,
942 mm, trigger);
943
944 else
945 return svm_migrate_vram_to_vram(prange, best_loc, start, last,
946 mm, trigger);
947
948 }
949
950 /**
951 * svm_migrate_to_ram - CPU page fault handler
952 * @vmf: CPU vm fault vma, address
953 *
954 * Context: vm fault handler, caller holds the mmap read lock
955 *
956 * Return:
957 * 0 - OK
958 * VM_FAULT_SIGBUS - notice application to have SIGBUS page fault
959 */
svm_migrate_to_ram(struct vm_fault * vmf)960 static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
961 {
962 unsigned long start, last, size;
963 unsigned long addr = vmf->address;
964 struct svm_range_bo *svm_bo;
965 struct svm_range *prange;
966 struct kfd_process *p;
967 struct mm_struct *mm;
968 int r = 0;
969
970 svm_bo = vmf->page->zone_device_data;
971 if (!svm_bo) {
972 pr_debug("failed get device page at addr 0x%lx\n", addr);
973 return VM_FAULT_SIGBUS;
974 }
975 if (!mmget_not_zero(svm_bo->mm)) {
976 pr_debug("addr 0x%lx of process mm is destroyed\n", addr);
977 return VM_FAULT_SIGBUS;
978 }
979
980 mm = svm_bo->mm;
981 if (mm != vmf->vma->vm_mm)
982 pr_debug("addr 0x%lx is COW mapping in child process\n", addr);
983
984 p = kfd_lookup_process_by_mm(mm);
985 if (!p) {
986 pr_debug("failed find process at fault address 0x%lx\n", addr);
987 r = VM_FAULT_SIGBUS;
988 goto out_mmput;
989 }
990 if (READ_ONCE(p->svms.faulting_task) == current) {
991 pr_debug("skipping ram migration\n");
992 r = 0;
993 goto out_unref_process;
994 }
995
996 pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr);
997 addr >>= PAGE_SHIFT;
998
999 mutex_lock(&p->svms.lock);
1000
1001 prange = svm_range_from_addr(&p->svms, addr, NULL);
1002 if (!prange) {
1003 pr_debug("failed get range svms 0x%p addr 0x%lx\n", &p->svms, addr);
1004 r = -EFAULT;
1005 goto out_unlock_svms;
1006 }
1007
1008 mutex_lock(&prange->migrate_mutex);
1009
1010 if (!prange->actual_loc)
1011 goto out_unlock_prange;
1012
1013 /* Align migration range start and size to granularity size */
1014 size = 1UL << prange->granularity;
1015 start = max(ALIGN_DOWN(addr, size), prange->start);
1016 last = min(ALIGN(addr + 1, size) - 1, prange->last);
1017
1018 r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, start, last,
1019 KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, vmf->page);
1020 if (r)
1021 pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",
1022 r, prange->svms, prange, start, last);
1023
1024 out_unlock_prange:
1025 mutex_unlock(&prange->migrate_mutex);
1026 out_unlock_svms:
1027 mutex_unlock(&p->svms.lock);
1028 out_unref_process:
1029 pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr);
1030 kfd_unref_process(p);
1031 out_mmput:
1032 mmput(mm);
1033 return r ? VM_FAULT_SIGBUS : 0;
1034 }
1035
1036 static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
1037 .folio_free = svm_migrate_folio_free,
1038 .migrate_to_ram = svm_migrate_to_ram,
1039 };
1040
1041 /* Each VRAM page uses sizeof(struct page) on system memory */
1042 #define SVM_HMM_PAGE_STRUCT_SIZE(size) ((size)/PAGE_SIZE * sizeof(struct page))
1043
kgd2kfd_init_zone_device(struct amdgpu_device * adev)1044 int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
1045 {
1046 struct amdgpu_kfd_dev *kfddev = &adev->kfd;
1047 struct dev_pagemap *pgmap;
1048 struct resource *res = NULL;
1049 unsigned long size;
1050 void *r;
1051
1052 /* Page migration works on gfx9 or newer */
1053 if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 0, 1))
1054 return -EINVAL;
1055
1056 if (adev->apu_prefer_gtt)
1057 return 0;
1058
1059 pgmap = &kfddev->pgmap;
1060 memset(pgmap, 0, sizeof(*pgmap));
1061
1062 /* TODO: register all vram to HMM for now.
1063 * should remove reserved size
1064 */
1065 size = ALIGN(adev->gmc.real_vram_size, 2ULL << 20);
1066 if (adev->gmc.xgmi.connected_to_cpu) {
1067 pgmap->range.start = adev->gmc.aper_base;
1068 pgmap->range.end = adev->gmc.aper_base + adev->gmc.aper_size - 1;
1069 pgmap->type = MEMORY_DEVICE_COHERENT;
1070 } else {
1071 res = devm_request_free_mem_region(adev->dev, &iomem_resource, size);
1072 if (IS_ERR(res))
1073 return PTR_ERR(res);
1074 pgmap->range.start = res->start;
1075 pgmap->range.end = res->end;
1076 pgmap->type = MEMORY_DEVICE_PRIVATE;
1077 }
1078
1079 pgmap->nr_range = 1;
1080 pgmap->ops = &svm_migrate_pgmap_ops;
1081 pgmap->owner = SVM_ADEV_PGMAP_OWNER(adev);
1082 pgmap->flags = 0;
1083 /* Device manager releases device-specific resources, memory region and
1084 * pgmap when driver disconnects from device.
1085 */
1086 r = devm_memremap_pages(adev->dev, pgmap);
1087 if (IS_ERR(r)) {
1088 pr_err("failed to register HMM device memory\n");
1089 if (pgmap->type == MEMORY_DEVICE_PRIVATE)
1090 devm_release_mem_region(adev->dev, res->start, resource_size(res));
1091 /* Disable SVM support capability */
1092 pgmap->type = 0;
1093 return PTR_ERR(r);
1094 }
1095
1096 pr_debug("reserve %ldMB system memory for VRAM pages struct\n",
1097 SVM_HMM_PAGE_STRUCT_SIZE(size) >> 20);
1098
1099 amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
1100
1101 pr_info("HMM registered %ldMB device memory\n", size >> 20);
1102
1103 return 0;
1104 }
1105