1 /*
2 * Copyright 2018 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "nouveau_dmem.h"
23 #include "nouveau_drv.h"
24 #include "nouveau_chan.h"
25 #include "nouveau_dma.h"
26 #include "nouveau_mem.h"
27 #include "nouveau_bo.h"
28 #include "nouveau_svm.h"
29
30 #include <nvif/class.h>
31 #include <nvif/object.h>
32 #include <nvif/push906f.h>
33 #include <nvif/if000c.h>
34 #include <nvif/if500b.h>
35 #include <nvif/if900b.h>
36
37 #include <nvhw/class/cla0b5.h>
38
39 #include <linux/sched/mm.h>
40 #include <linux/hmm.h>
41 #include <linux/memremap.h>
42 #include <linux/migrate.h>
43
44 /*
45 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
46 * it in vram while in use. We likely want to overhaul memory management for
47 * nouveau to be more page like (not necessarily with system page size but a
48 * bigger page size) at lowest level and have some shim layer on top that would
49 * provide the same functionality as TTM.
50 */
51 #define DMEM_CHUNK_SIZE (2UL << 20)
52 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
53 #define NR_CHUNKS (128)
54
55 enum nouveau_aper {
56 NOUVEAU_APER_VIRT,
57 NOUVEAU_APER_VRAM,
58 NOUVEAU_APER_HOST,
59 };
60
61 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
62 enum nouveau_aper, u64 dst_addr,
63 enum nouveau_aper, u64 src_addr);
64 typedef int (*nouveau_clear_page_t)(struct nouveau_drm *drm, u32 length,
65 enum nouveau_aper, u64 dst_addr);
66
67 struct nouveau_dmem_chunk {
68 struct list_head list;
69 struct nouveau_bo *bo;
70 struct nouveau_drm *drm;
71 unsigned long callocated;
72 struct dev_pagemap pagemap;
73 };
74
75 struct nouveau_dmem_migrate {
76 nouveau_migrate_copy_t copy_func;
77 nouveau_clear_page_t clear_func;
78 struct nouveau_channel *chan;
79 };
80
81 struct nouveau_dmem {
82 struct nouveau_drm *drm;
83 struct nouveau_dmem_migrate migrate;
84 struct list_head chunks;
85 struct mutex mutex;
86 struct page *free_pages;
87 struct folio *free_folios;
88 spinlock_t lock;
89 };
90
91 struct nouveau_dmem_dma_info {
92 dma_addr_t dma_addr;
93 size_t size;
94 };
95
nouveau_page_to_chunk(struct page * page)96 static struct nouveau_dmem_chunk *nouveau_page_to_chunk(struct page *page)
97 {
98 return container_of(page_pgmap(page), struct nouveau_dmem_chunk,
99 pagemap);
100 }
101
page_to_drm(struct page * page)102 static struct nouveau_drm *page_to_drm(struct page *page)
103 {
104 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page);
105
106 return chunk->drm;
107 }
108
nouveau_dmem_page_addr(struct page * page)109 unsigned long nouveau_dmem_page_addr(struct page *page)
110 {
111 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page);
112 unsigned long off = (page_to_pfn(page) << PAGE_SHIFT) -
113 chunk->pagemap.range.start;
114
115 return chunk->bo->offset + off;
116 }
117
nouveau_dmem_folio_free(struct folio * folio)118 static void nouveau_dmem_folio_free(struct folio *folio)
119 {
120 struct page *page = &folio->page;
121 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page);
122 struct nouveau_dmem *dmem = chunk->drm->dmem;
123
124 spin_lock(&dmem->lock);
125 if (folio_order(folio)) {
126 page->zone_device_data = dmem->free_folios;
127 dmem->free_folios = folio;
128 } else {
129 page->zone_device_data = dmem->free_pages;
130 dmem->free_pages = page;
131 }
132
133 WARN_ON(!chunk->callocated);
134 chunk->callocated--;
135 /*
136 * FIXME when chunk->callocated reach 0 we should add the chunk to
137 * a reclaim list so that it can be freed in case of memory pressure.
138 */
139 spin_unlock(&dmem->lock);
140 }
141
nouveau_dmem_fence_done(struct nouveau_fence ** fence)142 static void nouveau_dmem_fence_done(struct nouveau_fence **fence)
143 {
144 if (fence) {
145 nouveau_fence_wait(*fence, true, false);
146 nouveau_fence_unref(fence);
147 } else {
148 /*
149 * FIXME wait for channel to be IDLE before calling finalizing
150 * the hmem object.
151 */
152 }
153 }
154
nouveau_dmem_copy_folio(struct nouveau_drm * drm,struct folio * sfolio,struct folio * dfolio,struct nouveau_dmem_dma_info * dma_info)155 static int nouveau_dmem_copy_folio(struct nouveau_drm *drm,
156 struct folio *sfolio, struct folio *dfolio,
157 struct nouveau_dmem_dma_info *dma_info)
158 {
159 struct device *dev = drm->dev->dev;
160 struct page *dpage = folio_page(dfolio, 0);
161 struct page *spage = folio_page(sfolio, 0);
162
163 folio_lock(dfolio);
164
165 dma_info->dma_addr = dma_map_page(dev, dpage, 0, page_size(dpage),
166 DMA_BIDIRECTIONAL);
167 dma_info->size = page_size(dpage);
168 if (dma_mapping_error(dev, dma_info->dma_addr))
169 return -EIO;
170
171 if (drm->dmem->migrate.copy_func(drm, folio_nr_pages(sfolio),
172 NOUVEAU_APER_HOST, dma_info->dma_addr,
173 NOUVEAU_APER_VRAM,
174 nouveau_dmem_page_addr(spage))) {
175 dma_unmap_page(dev, dma_info->dma_addr, page_size(dpage),
176 DMA_BIDIRECTIONAL);
177 return -EIO;
178 }
179
180 return 0;
181 }
182
nouveau_dmem_migrate_to_ram(struct vm_fault * vmf)183 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
184 {
185 struct nouveau_drm *drm = page_to_drm(vmf->page);
186 struct nouveau_dmem *dmem = drm->dmem;
187 struct nouveau_fence *fence;
188 struct nouveau_svmm *svmm;
189 struct page *dpage;
190 vm_fault_t ret = 0;
191 int err;
192 struct migrate_vma args = {
193 .vma = vmf->vma,
194 .pgmap_owner = drm->dev,
195 .fault_page = vmf->page,
196 .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE |
197 MIGRATE_VMA_SELECT_COMPOUND,
198 .src = NULL,
199 .dst = NULL,
200 };
201 unsigned int order, nr;
202 struct folio *sfolio, *dfolio;
203 struct nouveau_dmem_dma_info dma_info;
204
205 sfolio = page_folio(vmf->page);
206 order = folio_order(sfolio);
207 nr = 1 << order;
208
209 /*
210 * Handle partial unmap faults, where the folio is large, but
211 * the pmd is split.
212 */
213 if (vmf->pte) {
214 order = 0;
215 nr = 1;
216 }
217
218 if (order)
219 args.flags |= MIGRATE_VMA_SELECT_COMPOUND;
220
221 args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order));
222 args.vma = vmf->vma;
223 args.end = args.start + (PAGE_SIZE << order);
224 args.src = kcalloc(nr, sizeof(*args.src), GFP_KERNEL);
225 args.dst = kcalloc(nr, sizeof(*args.dst), GFP_KERNEL);
226
227 if (!args.src || !args.dst) {
228 ret = VM_FAULT_OOM;
229 goto err;
230 }
231 /*
232 * FIXME what we really want is to find some heuristic to migrate more
233 * than just one page on CPU fault. When such fault happens it is very
234 * likely that more surrounding page will CPU fault too.
235 */
236 if (migrate_vma_setup(&args) < 0)
237 return VM_FAULT_SIGBUS;
238 if (!args.cpages)
239 return 0;
240
241 if (order)
242 dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER | __GFP_ZERO,
243 order, vmf->vma, vmf->address), 0);
244 else
245 dpage = alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vmf->vma,
246 vmf->address);
247 if (!dpage) {
248 ret = VM_FAULT_OOM;
249 goto done;
250 }
251
252 args.dst[0] = migrate_pfn(page_to_pfn(dpage));
253 if (order)
254 args.dst[0] |= MIGRATE_PFN_COMPOUND;
255 dfolio = page_folio(dpage);
256
257 svmm = folio_zone_device_data(sfolio);
258 mutex_lock(&svmm->mutex);
259 nouveau_svmm_invalidate(svmm, args.start, args.end);
260 err = nouveau_dmem_copy_folio(drm, sfolio, dfolio, &dma_info);
261 mutex_unlock(&svmm->mutex);
262 if (err) {
263 ret = VM_FAULT_SIGBUS;
264 goto done;
265 }
266
267 nouveau_fence_new(&fence, dmem->migrate.chan);
268 migrate_vma_pages(&args);
269 nouveau_dmem_fence_done(&fence);
270 dma_unmap_page(drm->dev->dev, dma_info.dma_addr, dma_info.size,
271 DMA_BIDIRECTIONAL);
272 done:
273 migrate_vma_finalize(&args);
274 err:
275 kfree(args.src);
276 kfree(args.dst);
277 return ret;
278 }
279
nouveau_dmem_folio_split(struct folio * head,struct folio * tail)280 static void nouveau_dmem_folio_split(struct folio *head, struct folio *tail)
281 {
282 struct nouveau_dmem_chunk *chunk;
283 struct nouveau_dmem *dmem;
284
285 if (tail == NULL)
286 return;
287 tail->pgmap = head->pgmap;
288 tail->mapping = head->mapping;
289 folio_set_zone_device_data(tail, folio_zone_device_data(head));
290
291 /*
292 * The split hands out a new independently-freeable folio that will
293 * later be released via nouveau_dmem_folio_free(); account for it so
294 * chunk->callocated stays balanced.
295 */
296 chunk = nouveau_page_to_chunk(&head->page);
297 dmem = chunk->drm->dmem;
298 spin_lock(&dmem->lock);
299 chunk->callocated++;
300 spin_unlock(&dmem->lock);
301 }
302
303 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
304 .folio_free = nouveau_dmem_folio_free,
305 .migrate_to_ram = nouveau_dmem_migrate_to_ram,
306 .folio_split = nouveau_dmem_folio_split,
307 };
308
309 static int
nouveau_dmem_chunk_alloc(struct nouveau_drm * drm,struct page ** ppage,bool is_large)310 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage,
311 bool is_large)
312 {
313 struct nouveau_dmem_chunk *chunk;
314 struct resource *res;
315 struct page *page;
316 void *ptr;
317 unsigned long i, pfn_first, pfn;
318 int ret;
319
320 chunk = kzalloc_obj(*chunk);
321 if (chunk == NULL) {
322 ret = -ENOMEM;
323 goto out;
324 }
325
326 /* Allocate unused physical address space for device private pages. */
327 res = request_free_mem_region(&iomem_resource, DMEM_CHUNK_SIZE * NR_CHUNKS,
328 "nouveau_dmem");
329 if (IS_ERR(res)) {
330 ret = PTR_ERR(res);
331 goto out_free;
332 }
333
334 chunk->drm = drm;
335 chunk->pagemap.type = MEMORY_DEVICE_PRIVATE;
336 chunk->pagemap.range.start = res->start;
337 chunk->pagemap.range.end = res->end;
338 chunk->pagemap.nr_range = 1;
339 chunk->pagemap.ops = &nouveau_dmem_pagemap_ops;
340 chunk->pagemap.owner = drm->dev;
341
342 ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM,
343 DMEM_CHUNK_SIZE * NR_CHUNKS, &chunk->bo);
344 if (ret)
345 goto out_release;
346
347 ptr = memremap_pages(&chunk->pagemap, numa_node_id());
348 if (IS_ERR(ptr)) {
349 ret = PTR_ERR(ptr);
350 goto out_bo_free;
351 }
352
353 mutex_lock(&drm->dmem->mutex);
354 list_add(&chunk->list, &drm->dmem->chunks);
355 mutex_unlock(&drm->dmem->mutex);
356
357 pfn_first = chunk->pagemap.range.start >> PAGE_SHIFT;
358 page = pfn_to_page(pfn_first);
359 spin_lock(&drm->dmem->lock);
360
361 pfn = pfn_first;
362 for (i = 0; i < NR_CHUNKS; i++) {
363 int j;
364
365 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || !is_large) {
366 for (j = 0; j < DMEM_CHUNK_NPAGES - 1; j++, pfn++) {
367 page = pfn_to_page(pfn);
368 page->zone_device_data = drm->dmem->free_pages;
369 drm->dmem->free_pages = page;
370 }
371 } else {
372 page = pfn_to_page(pfn);
373 page->zone_device_data = drm->dmem->free_folios;
374 drm->dmem->free_folios = page_folio(page);
375 pfn += DMEM_CHUNK_NPAGES;
376 }
377 }
378
379 /* Move to next page */
380 if (is_large) {
381 *ppage = &drm->dmem->free_folios->page;
382 drm->dmem->free_folios = (*ppage)->zone_device_data;
383 } else {
384 *ppage = drm->dmem->free_pages;
385 drm->dmem->free_pages = (*ppage)->zone_device_data;
386 }
387
388 chunk->callocated++;
389 spin_unlock(&drm->dmem->lock);
390
391 NV_INFO(drm, "DMEM: registered %ldMB of %sdevice memory %lx %lx\n",
392 NR_CHUNKS * DMEM_CHUNK_SIZE >> 20, is_large ? "THP " : "", pfn_first,
393 nouveau_dmem_page_addr(page));
394
395 return 0;
396
397 out_bo_free:
398 nouveau_bo_unpin_del(&chunk->bo);
399 out_release:
400 release_mem_region(chunk->pagemap.range.start, range_len(&chunk->pagemap.range));
401 out_free:
402 kfree(chunk);
403 out:
404 return ret;
405 }
406
407 static struct page *
nouveau_dmem_page_alloc_locked(struct nouveau_drm * drm,bool is_large)408 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm, bool is_large)
409 {
410 struct nouveau_dmem_chunk *chunk;
411 struct page *page = NULL;
412 struct folio *folio = NULL;
413 int ret;
414 unsigned int order = 0;
415
416 spin_lock(&drm->dmem->lock);
417 if (is_large && drm->dmem->free_folios) {
418 folio = drm->dmem->free_folios;
419 page = &folio->page;
420 drm->dmem->free_folios = page->zone_device_data;
421 chunk = nouveau_page_to_chunk(&folio->page);
422 chunk->callocated++;
423 spin_unlock(&drm->dmem->lock);
424 order = ilog2(DMEM_CHUNK_NPAGES);
425 } else if (!is_large && drm->dmem->free_pages) {
426 page = drm->dmem->free_pages;
427 drm->dmem->free_pages = page->zone_device_data;
428 chunk = nouveau_page_to_chunk(page);
429 chunk->callocated++;
430 spin_unlock(&drm->dmem->lock);
431 folio = page_folio(page);
432 } else {
433 spin_unlock(&drm->dmem->lock);
434 ret = nouveau_dmem_chunk_alloc(drm, &page, is_large);
435 if (ret)
436 return NULL;
437 folio = page_folio(page);
438 if (is_large)
439 order = ilog2(DMEM_CHUNK_NPAGES);
440 }
441
442 zone_device_folio_init(folio, page_pgmap(folio_page(folio, 0)), order);
443 return page;
444 }
445
446 static void
nouveau_dmem_page_free_locked(struct nouveau_drm * drm,struct page * page)447 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
448 {
449 unlock_page(page);
450 put_page(page);
451 }
452
453 void
nouveau_dmem_resume(struct nouveau_drm * drm)454 nouveau_dmem_resume(struct nouveau_drm *drm)
455 {
456 struct nouveau_dmem_chunk *chunk;
457 int ret;
458
459 if (drm->dmem == NULL)
460 return;
461
462 mutex_lock(&drm->dmem->mutex);
463 list_for_each_entry(chunk, &drm->dmem->chunks, list) {
464 ret = nouveau_bo_pin(chunk->bo, NOUVEAU_GEM_DOMAIN_VRAM, false);
465 /* FIXME handle pin failure */
466 WARN_ON(ret);
467 }
468 mutex_unlock(&drm->dmem->mutex);
469 }
470
471 void
nouveau_dmem_suspend(struct nouveau_drm * drm)472 nouveau_dmem_suspend(struct nouveau_drm *drm)
473 {
474 struct nouveau_dmem_chunk *chunk;
475
476 if (drm->dmem == NULL)
477 return;
478
479 mutex_lock(&drm->dmem->mutex);
480 list_for_each_entry(chunk, &drm->dmem->chunks, list)
481 nouveau_bo_unpin(chunk->bo);
482 mutex_unlock(&drm->dmem->mutex);
483 }
484
485 /*
486 * Evict all pages mapping a chunk.
487 */
488 static void
nouveau_dmem_evict_chunk(struct nouveau_dmem_chunk * chunk)489 nouveau_dmem_evict_chunk(struct nouveau_dmem_chunk *chunk)
490 {
491 unsigned long i, npages = range_len(&chunk->pagemap.range) >> PAGE_SHIFT;
492 unsigned long *src_pfns, *dst_pfns;
493 struct nouveau_dmem_dma_info *dma_info;
494 struct nouveau_fence *fence;
495
496 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
497 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
498 dma_info = kvzalloc_objs(*dma_info, npages, GFP_KERNEL | __GFP_NOFAIL);
499
500 migrate_device_range(src_pfns, chunk->pagemap.range.start >> PAGE_SHIFT,
501 npages);
502
503 for (i = 0; i < npages; i++) {
504 if (src_pfns[i] & MIGRATE_PFN_MIGRATE) {
505 struct page *dpage;
506 struct folio *folio = page_folio(
507 migrate_pfn_to_page(src_pfns[i]));
508 unsigned int order = folio_order(folio);
509
510 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) {
511 dpage = folio_page(
512 folio_alloc(
513 GFP_HIGHUSER_MOVABLE, order), 0);
514 } else {
515 /*
516 * _GFP_NOFAIL because the GPU is going away and there
517 * is nothing sensible we can do if we can't copy the
518 * data back.
519 */
520 dpage = alloc_page(GFP_HIGHUSER | __GFP_NOFAIL);
521 }
522
523 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
524 nouveau_dmem_copy_folio(chunk->drm,
525 page_folio(migrate_pfn_to_page(src_pfns[i])),
526 page_folio(dpage),
527 &dma_info[i]);
528 }
529 }
530
531 nouveau_fence_new(&fence, chunk->drm->dmem->migrate.chan);
532 migrate_device_pages(src_pfns, dst_pfns, npages);
533 nouveau_dmem_fence_done(&fence);
534 migrate_device_finalize(src_pfns, dst_pfns, npages);
535 kvfree(src_pfns);
536 kvfree(dst_pfns);
537 for (i = 0; i < npages; i++)
538 dma_unmap_page(chunk->drm->dev->dev, dma_info[i].dma_addr,
539 dma_info[i].size, DMA_BIDIRECTIONAL);
540 kvfree(dma_info);
541 }
542
543 void
nouveau_dmem_fini(struct nouveau_drm * drm)544 nouveau_dmem_fini(struct nouveau_drm *drm)
545 {
546 struct nouveau_dmem_chunk *chunk, *tmp;
547
548 if (drm->dmem == NULL)
549 return;
550
551 mutex_lock(&drm->dmem->mutex);
552
553 list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) {
554 nouveau_dmem_evict_chunk(chunk);
555 nouveau_bo_unpin_del(&chunk->bo);
556 WARN_ON(chunk->callocated);
557 list_del(&chunk->list);
558 memunmap_pages(&chunk->pagemap);
559 release_mem_region(chunk->pagemap.range.start,
560 range_len(&chunk->pagemap.range));
561 kfree(chunk);
562 }
563
564 mutex_unlock(&drm->dmem->mutex);
565 }
566
567 static int
nvc0b5_migrate_copy(struct nouveau_drm * drm,u64 npages,enum nouveau_aper dst_aper,u64 dst_addr,enum nouveau_aper src_aper,u64 src_addr)568 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
569 enum nouveau_aper dst_aper, u64 dst_addr,
570 enum nouveau_aper src_aper, u64 src_addr)
571 {
572 struct nvif_push *push = &drm->dmem->migrate.chan->chan.push;
573 u32 launch_dma = 0;
574 int ret;
575
576 ret = PUSH_WAIT(push, 13);
577 if (ret)
578 return ret;
579
580 if (src_aper != NOUVEAU_APER_VIRT) {
581 switch (src_aper) {
582 case NOUVEAU_APER_VRAM:
583 PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE,
584 NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, LOCAL_FB));
585 break;
586 case NOUVEAU_APER_HOST:
587 PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE,
588 NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, COHERENT_SYSMEM));
589 break;
590 default:
591 return -EINVAL;
592 }
593
594 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, SRC_TYPE, PHYSICAL);
595 }
596
597 if (dst_aper != NOUVEAU_APER_VIRT) {
598 switch (dst_aper) {
599 case NOUVEAU_APER_VRAM:
600 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
601 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB));
602 break;
603 case NOUVEAU_APER_HOST:
604 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
605 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM));
606 break;
607 default:
608 return -EINVAL;
609 }
610
611 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL);
612 }
613
614 PUSH_MTHD(push, NVA0B5, OFFSET_IN_UPPER,
615 NVVAL(NVA0B5, OFFSET_IN_UPPER, UPPER, upper_32_bits(src_addr)),
616
617 OFFSET_IN_LOWER, lower_32_bits(src_addr),
618
619 OFFSET_OUT_UPPER,
620 NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)),
621
622 OFFSET_OUT_LOWER, lower_32_bits(dst_addr),
623 PITCH_IN, PAGE_SIZE,
624 PITCH_OUT, PAGE_SIZE,
625 LINE_LENGTH_IN, PAGE_SIZE,
626 LINE_COUNT, npages);
627
628 PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma |
629 NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) |
630 NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) |
631 NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) |
632 NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) |
633 NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) |
634 NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) |
635 NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, TRUE) |
636 NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, FALSE) |
637 NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING));
638 return 0;
639 }
640
641 static int
nvc0b5_migrate_clear(struct nouveau_drm * drm,u32 length,enum nouveau_aper dst_aper,u64 dst_addr)642 nvc0b5_migrate_clear(struct nouveau_drm *drm, u32 length,
643 enum nouveau_aper dst_aper, u64 dst_addr)
644 {
645 struct nvif_push *push = &drm->dmem->migrate.chan->chan.push;
646 u32 launch_dma = 0;
647 int ret;
648
649 ret = PUSH_WAIT(push, 12);
650 if (ret)
651 return ret;
652
653 switch (dst_aper) {
654 case NOUVEAU_APER_VRAM:
655 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
656 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB));
657 break;
658 case NOUVEAU_APER_HOST:
659 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
660 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM));
661 break;
662 default:
663 return -EINVAL;
664 }
665
666 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL);
667
668 PUSH_MTHD(push, NVA0B5, SET_REMAP_CONST_A, 0,
669 SET_REMAP_CONST_B, 0,
670
671 SET_REMAP_COMPONENTS,
672 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_X, CONST_A) |
673 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_Y, CONST_B) |
674 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, COMPONENT_SIZE, FOUR) |
675 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, NUM_DST_COMPONENTS, TWO));
676
677 PUSH_MTHD(push, NVA0B5, OFFSET_OUT_UPPER,
678 NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)),
679
680 OFFSET_OUT_LOWER, lower_32_bits(dst_addr));
681
682 PUSH_MTHD(push, NVA0B5, LINE_LENGTH_IN, length >> 3);
683
684 PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma |
685 NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) |
686 NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) |
687 NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) |
688 NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) |
689 NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) |
690 NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) |
691 NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, FALSE) |
692 NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, TRUE) |
693 NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING));
694 return 0;
695 }
696
697 static int
nouveau_dmem_migrate_init(struct nouveau_drm * drm)698 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
699 {
700 switch (drm->ttm.copy.oclass) {
701 case PASCAL_DMA_COPY_A:
702 case PASCAL_DMA_COPY_B:
703 case VOLTA_DMA_COPY_A:
704 case TURING_DMA_COPY_A:
705 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
706 drm->dmem->migrate.clear_func = nvc0b5_migrate_clear;
707 drm->dmem->migrate.chan = drm->ttm.chan;
708 return 0;
709 default:
710 break;
711 }
712 return -ENODEV;
713 }
714
715 void
nouveau_dmem_init(struct nouveau_drm * drm)716 nouveau_dmem_init(struct nouveau_drm *drm)
717 {
718 int ret;
719
720 /* This only make sense on PASCAL or newer */
721 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
722 return;
723
724 if (!(drm->dmem = kzalloc_obj(*drm->dmem)))
725 return;
726
727 drm->dmem->drm = drm;
728 mutex_init(&drm->dmem->mutex);
729 INIT_LIST_HEAD(&drm->dmem->chunks);
730 mutex_init(&drm->dmem->mutex);
731 spin_lock_init(&drm->dmem->lock);
732
733 /* Initialize migration dma helpers before registering memory */
734 ret = nouveau_dmem_migrate_init(drm);
735 if (ret) {
736 kfree(drm->dmem);
737 drm->dmem = NULL;
738 }
739 }
740
nouveau_dmem_migrate_copy_one(struct nouveau_drm * drm,struct nouveau_svmm * svmm,unsigned long src,struct nouveau_dmem_dma_info * dma_info,u64 * pfn)741 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,
742 struct nouveau_svmm *svmm, unsigned long src,
743 struct nouveau_dmem_dma_info *dma_info, u64 *pfn)
744 {
745 struct device *dev = drm->dev->dev;
746 struct page *dpage, *spage;
747 unsigned long paddr;
748 bool is_large = false;
749 unsigned long mpfn;
750
751 spage = migrate_pfn_to_page(src);
752 if (!(src & MIGRATE_PFN_MIGRATE))
753 goto out;
754
755 is_large = src & MIGRATE_PFN_COMPOUND;
756 dpage = nouveau_dmem_page_alloc_locked(drm, is_large);
757 if (!dpage)
758 goto out;
759
760 paddr = nouveau_dmem_page_addr(dpage);
761 if (spage) {
762 dma_info->dma_addr = dma_map_page(dev, spage, 0, page_size(spage),
763 DMA_BIDIRECTIONAL);
764 dma_info->size = page_size(spage);
765 if (dma_mapping_error(dev, dma_info->dma_addr))
766 goto out_free_page;
767 if (drm->dmem->migrate.copy_func(drm, folio_nr_pages(page_folio(spage)),
768 NOUVEAU_APER_VRAM, paddr, NOUVEAU_APER_HOST,
769 dma_info->dma_addr))
770 goto out_dma_unmap;
771 } else {
772 dma_info->dma_addr = DMA_MAPPING_ERROR;
773 if (drm->dmem->migrate.clear_func(drm, page_size(dpage),
774 NOUVEAU_APER_VRAM, paddr))
775 goto out_free_page;
776 }
777
778 dpage->zone_device_data = svmm;
779 *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM |
780 ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT);
781 if (src & MIGRATE_PFN_WRITE)
782 *pfn |= NVIF_VMM_PFNMAP_V0_W;
783 mpfn = migrate_pfn(page_to_pfn(dpage));
784 if (folio_order(page_folio(dpage)))
785 mpfn |= MIGRATE_PFN_COMPOUND;
786 return mpfn;
787
788 out_dma_unmap:
789 dma_unmap_page(dev, dma_info->dma_addr, dma_info->size, DMA_BIDIRECTIONAL);
790 out_free_page:
791 nouveau_dmem_page_free_locked(drm, dpage);
792 out:
793 *pfn = NVIF_VMM_PFNMAP_V0_NONE;
794 return 0;
795 }
796
nouveau_dmem_migrate_chunk(struct nouveau_drm * drm,struct nouveau_svmm * svmm,struct migrate_vma * args,struct nouveau_dmem_dma_info * dma_info,u64 * pfns)797 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm,
798 struct nouveau_svmm *svmm, struct migrate_vma *args,
799 struct nouveau_dmem_dma_info *dma_info, u64 *pfns)
800 {
801 struct nouveau_fence *fence;
802 unsigned long addr = args->start, nr_dma = 0, i;
803 unsigned long order = 0;
804
805 for (i = 0; addr < args->end; ) {
806 struct folio *folio;
807
808 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, svmm,
809 args->src[i], dma_info + nr_dma, pfns + i);
810 if (!args->dst[i]) {
811 i++;
812 addr += PAGE_SIZE;
813 continue;
814 }
815 if (!dma_mapping_error(drm->dev->dev, dma_info[nr_dma].dma_addr))
816 nr_dma++;
817 folio = page_folio(migrate_pfn_to_page(args->dst[i]));
818 order = folio_order(folio);
819 i += 1 << order;
820 addr += (1 << order) * PAGE_SIZE;
821 }
822
823 nouveau_fence_new(&fence, drm->dmem->migrate.chan);
824 migrate_vma_pages(args);
825 nouveau_dmem_fence_done(&fence);
826 nouveau_pfns_map(svmm, args->vma->vm_mm, args->start, pfns, i, order);
827
828 while (nr_dma--) {
829 dma_unmap_page(drm->dev->dev, dma_info[nr_dma].dma_addr,
830 dma_info[nr_dma].size, DMA_BIDIRECTIONAL);
831 }
832 migrate_vma_finalize(args);
833 }
834
835 int
nouveau_dmem_migrate_vma(struct nouveau_drm * drm,struct nouveau_svmm * svmm,struct vm_area_struct * vma,unsigned long start,unsigned long end)836 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
837 struct nouveau_svmm *svmm,
838 struct vm_area_struct *vma,
839 unsigned long start,
840 unsigned long end)
841 {
842 unsigned long npages = (end - start) >> PAGE_SHIFT;
843 unsigned long max = npages;
844 struct migrate_vma args = {
845 .vma = vma,
846 .start = start,
847 .pgmap_owner = drm->dev,
848 .flags = MIGRATE_VMA_SELECT_SYSTEM
849 | MIGRATE_VMA_SELECT_COMPOUND,
850 };
851 unsigned long i;
852 u64 *pfns;
853 int ret = -ENOMEM;
854 struct nouveau_dmem_dma_info *dma_info;
855
856 if (drm->dmem == NULL) {
857 ret = -ENODEV;
858 goto out;
859 }
860
861 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
862 if (max > (unsigned long)HPAGE_PMD_NR)
863 max = (unsigned long)HPAGE_PMD_NR;
864
865 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL);
866 if (!args.src)
867 goto out;
868 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL);
869 if (!args.dst)
870 goto out_free_src;
871
872 dma_info = kmalloc_objs(*dma_info, max);
873 if (!dma_info)
874 goto out_free_dst;
875
876 pfns = nouveau_pfns_alloc(max);
877 if (!pfns)
878 goto out_free_dma;
879
880 for (i = 0; i < npages; i += max) {
881 if (args.start + (max << PAGE_SHIFT) > end)
882 args.end = end;
883 else
884 args.end = args.start + (max << PAGE_SHIFT);
885
886 ret = migrate_vma_setup(&args);
887 if (ret)
888 goto out_free_pfns;
889
890 if (args.cpages)
891 nouveau_dmem_migrate_chunk(drm, svmm, &args, dma_info,
892 pfns);
893 args.start = args.end;
894 }
895
896 ret = 0;
897 out_free_pfns:
898 nouveau_pfns_free(pfns);
899 out_free_dma:
900 kfree(dma_info);
901 out_free_dst:
902 kfree(args.dst);
903 out_free_src:
904 kfree(args.src);
905 out:
906 return ret;
907 }
908