1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_dmem.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dma.h" 26 #include "nouveau_mem.h" 27 #include "nouveau_bo.h" 28 #include "nouveau_svm.h" 29 30 #include <nvif/class.h> 31 #include <nvif/object.h> 32 #include <nvif/if000c.h> 33 #include <nvif/if500b.h> 34 #include <nvif/if900b.h> 35 #include <nvif/if000c.h> 36 37 #include <linux/sched/mm.h> 38 #include <linux/hmm.h> 39 40 /* 41 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 42 * it in vram while in use. We likely want to overhaul memory management for 43 * nouveau to be more page like (not necessarily with system page size but a 44 * bigger page size) at lowest level and have some shim layer on top that would 45 * provide the same functionality as TTM. 46 */ 47 #define DMEM_CHUNK_SIZE (2UL << 20) 48 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 49 50 enum nouveau_aper { 51 NOUVEAU_APER_VIRT, 52 NOUVEAU_APER_VRAM, 53 NOUVEAU_APER_HOST, 54 }; 55 56 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 57 enum nouveau_aper, u64 dst_addr, 58 enum nouveau_aper, u64 src_addr); 59 typedef int (*nouveau_clear_page_t)(struct nouveau_drm *drm, u32 length, 60 enum nouveau_aper, u64 dst_addr); 61 62 struct nouveau_dmem_chunk { 63 struct list_head list; 64 struct nouveau_bo *bo; 65 struct nouveau_drm *drm; 66 unsigned long callocated; 67 struct dev_pagemap pagemap; 68 }; 69 70 struct nouveau_dmem_migrate { 71 nouveau_migrate_copy_t copy_func; 72 nouveau_clear_page_t clear_func; 73 struct nouveau_channel *chan; 74 }; 75 76 struct nouveau_dmem { 77 struct nouveau_drm *drm; 78 struct nouveau_dmem_migrate migrate; 79 struct list_head chunks; 80 struct mutex mutex; 81 struct page *free_pages; 82 spinlock_t lock; 83 }; 84 85 static struct nouveau_dmem_chunk *nouveau_page_to_chunk(struct page *page) 86 { 87 return container_of(page->pgmap, struct nouveau_dmem_chunk, pagemap); 88 } 89 90 static struct nouveau_drm *page_to_drm(struct page *page) 91 { 92 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 93 94 return chunk->drm; 95 } 96 97 unsigned long nouveau_dmem_page_addr(struct page *page) 98 { 99 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 100 unsigned long off = (page_to_pfn(page) << PAGE_SHIFT) - 101 chunk->pagemap.res.start; 102 103 return chunk->bo->bo.offset + off; 104 } 105 106 static void nouveau_dmem_page_free(struct page *page) 107 { 108 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 109 struct nouveau_dmem *dmem = chunk->drm->dmem; 110 111 spin_lock(&dmem->lock); 112 page->zone_device_data = dmem->free_pages; 113 dmem->free_pages = page; 114 115 WARN_ON(!chunk->callocated); 116 chunk->callocated--; 117 /* 118 * FIXME when chunk->callocated reach 0 we should add the chunk to 119 * a reclaim list so that it can be freed in case of memory pressure. 120 */ 121 spin_unlock(&dmem->lock); 122 } 123 124 static void nouveau_dmem_fence_done(struct nouveau_fence **fence) 125 { 126 if (fence) { 127 nouveau_fence_wait(*fence, true, false); 128 nouveau_fence_unref(fence); 129 } else { 130 /* 131 * FIXME wait for channel to be IDLE before calling finalizing 132 * the hmem object. 133 */ 134 } 135 } 136 137 static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, 138 struct vm_fault *vmf, struct migrate_vma *args, 139 dma_addr_t *dma_addr) 140 { 141 struct device *dev = drm->dev->dev; 142 struct page *dpage, *spage; 143 144 spage = migrate_pfn_to_page(args->src[0]); 145 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE)) 146 return 0; 147 148 dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address); 149 if (!dpage) 150 return VM_FAULT_SIGBUS; 151 lock_page(dpage); 152 153 *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 154 if (dma_mapping_error(dev, *dma_addr)) 155 goto error_free_page; 156 157 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr, 158 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) 159 goto error_dma_unmap; 160 161 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 162 return 0; 163 164 error_dma_unmap: 165 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 166 error_free_page: 167 __free_page(dpage); 168 return VM_FAULT_SIGBUS; 169 } 170 171 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) 172 { 173 struct nouveau_drm *drm = page_to_drm(vmf->page); 174 struct nouveau_dmem *dmem = drm->dmem; 175 struct nouveau_fence *fence; 176 unsigned long src = 0, dst = 0; 177 dma_addr_t dma_addr = 0; 178 vm_fault_t ret; 179 struct migrate_vma args = { 180 .vma = vmf->vma, 181 .start = vmf->address, 182 .end = vmf->address + PAGE_SIZE, 183 .src = &src, 184 .dst = &dst, 185 .src_owner = drm->dev, 186 }; 187 188 /* 189 * FIXME what we really want is to find some heuristic to migrate more 190 * than just one page on CPU fault. When such fault happens it is very 191 * likely that more surrounding page will CPU fault too. 192 */ 193 if (migrate_vma_setup(&args) < 0) 194 return VM_FAULT_SIGBUS; 195 if (!args.cpages) 196 return 0; 197 198 ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr); 199 if (ret || dst == 0) 200 goto done; 201 202 nouveau_fence_new(dmem->migrate.chan, false, &fence); 203 migrate_vma_pages(&args); 204 nouveau_dmem_fence_done(&fence); 205 dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 206 done: 207 migrate_vma_finalize(&args); 208 return ret; 209 } 210 211 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { 212 .page_free = nouveau_dmem_page_free, 213 .migrate_to_ram = nouveau_dmem_migrate_to_ram, 214 }; 215 216 static int 217 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage) 218 { 219 struct nouveau_dmem_chunk *chunk; 220 struct resource *res; 221 struct page *page; 222 void *ptr; 223 unsigned long i, pfn_first; 224 int ret; 225 226 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 227 if (chunk == NULL) { 228 ret = -ENOMEM; 229 goto out; 230 } 231 232 /* Allocate unused physical address space for device private pages. */ 233 res = request_free_mem_region(&iomem_resource, DMEM_CHUNK_SIZE, 234 "nouveau_dmem"); 235 if (IS_ERR(res)) { 236 ret = PTR_ERR(res); 237 goto out_free; 238 } 239 240 chunk->drm = drm; 241 chunk->pagemap.type = MEMORY_DEVICE_PRIVATE; 242 chunk->pagemap.res = *res; 243 chunk->pagemap.ops = &nouveau_dmem_pagemap_ops; 244 chunk->pagemap.owner = drm->dev; 245 246 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, 247 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, 248 &chunk->bo); 249 if (ret) 250 goto out_release; 251 252 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 253 if (ret) 254 goto out_bo_free; 255 256 ptr = memremap_pages(&chunk->pagemap, numa_node_id()); 257 if (IS_ERR(ptr)) { 258 ret = PTR_ERR(ptr); 259 goto out_bo_unpin; 260 } 261 262 mutex_lock(&drm->dmem->mutex); 263 list_add(&chunk->list, &drm->dmem->chunks); 264 mutex_unlock(&drm->dmem->mutex); 265 266 pfn_first = chunk->pagemap.res.start >> PAGE_SHIFT; 267 page = pfn_to_page(pfn_first); 268 spin_lock(&drm->dmem->lock); 269 for (i = 0; i < DMEM_CHUNK_NPAGES - 1; ++i, ++page) { 270 page->zone_device_data = drm->dmem->free_pages; 271 drm->dmem->free_pages = page; 272 } 273 *ppage = page; 274 chunk->callocated++; 275 spin_unlock(&drm->dmem->lock); 276 277 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", 278 DMEM_CHUNK_SIZE >> 20); 279 280 return 0; 281 282 out_bo_unpin: 283 nouveau_bo_unpin(chunk->bo); 284 out_bo_free: 285 nouveau_bo_ref(NULL, &chunk->bo); 286 out_release: 287 release_mem_region(chunk->pagemap.res.start, 288 resource_size(&chunk->pagemap.res)); 289 out_free: 290 kfree(chunk); 291 out: 292 return ret; 293 } 294 295 static struct page * 296 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 297 { 298 struct nouveau_dmem_chunk *chunk; 299 struct page *page = NULL; 300 int ret; 301 302 spin_lock(&drm->dmem->lock); 303 if (drm->dmem->free_pages) { 304 page = drm->dmem->free_pages; 305 drm->dmem->free_pages = page->zone_device_data; 306 chunk = nouveau_page_to_chunk(page); 307 chunk->callocated++; 308 spin_unlock(&drm->dmem->lock); 309 } else { 310 spin_unlock(&drm->dmem->lock); 311 ret = nouveau_dmem_chunk_alloc(drm, &page); 312 if (ret) 313 return NULL; 314 } 315 316 get_page(page); 317 lock_page(page); 318 return page; 319 } 320 321 static void 322 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 323 { 324 unlock_page(page); 325 put_page(page); 326 } 327 328 void 329 nouveau_dmem_resume(struct nouveau_drm *drm) 330 { 331 struct nouveau_dmem_chunk *chunk; 332 int ret; 333 334 if (drm->dmem == NULL) 335 return; 336 337 mutex_lock(&drm->dmem->mutex); 338 list_for_each_entry(chunk, &drm->dmem->chunks, list) { 339 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 340 /* FIXME handle pin failure */ 341 WARN_ON(ret); 342 } 343 mutex_unlock(&drm->dmem->mutex); 344 } 345 346 void 347 nouveau_dmem_suspend(struct nouveau_drm *drm) 348 { 349 struct nouveau_dmem_chunk *chunk; 350 351 if (drm->dmem == NULL) 352 return; 353 354 mutex_lock(&drm->dmem->mutex); 355 list_for_each_entry(chunk, &drm->dmem->chunks, list) 356 nouveau_bo_unpin(chunk->bo); 357 mutex_unlock(&drm->dmem->mutex); 358 } 359 360 void 361 nouveau_dmem_fini(struct nouveau_drm *drm) 362 { 363 struct nouveau_dmem_chunk *chunk, *tmp; 364 365 if (drm->dmem == NULL) 366 return; 367 368 mutex_lock(&drm->dmem->mutex); 369 370 list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) { 371 nouveau_bo_unpin(chunk->bo); 372 nouveau_bo_ref(NULL, &chunk->bo); 373 list_del(&chunk->list); 374 memunmap_pages(&chunk->pagemap); 375 release_mem_region(chunk->pagemap.res.start, 376 resource_size(&chunk->pagemap.res)); 377 kfree(chunk); 378 } 379 380 mutex_unlock(&drm->dmem->mutex); 381 } 382 383 static int 384 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 385 enum nouveau_aper dst_aper, u64 dst_addr, 386 enum nouveau_aper src_aper, u64 src_addr) 387 { 388 struct nouveau_channel *chan = drm->dmem->migrate.chan; 389 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ | 390 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | 391 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | 392 (1 << 2) /* FLUSH_ENABLE_TRUE. */ | 393 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; 394 int ret; 395 396 ret = RING_SPACE(chan, 13); 397 if (ret) 398 return ret; 399 400 if (src_aper != NOUVEAU_APER_VIRT) { 401 switch (src_aper) { 402 case NOUVEAU_APER_VRAM: 403 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0); 404 break; 405 case NOUVEAU_APER_HOST: 406 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1); 407 break; 408 default: 409 return -EINVAL; 410 } 411 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */ 412 } 413 414 if (dst_aper != NOUVEAU_APER_VIRT) { 415 switch (dst_aper) { 416 case NOUVEAU_APER_VRAM: 417 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); 418 break; 419 case NOUVEAU_APER_HOST: 420 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); 421 break; 422 default: 423 return -EINVAL; 424 } 425 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ 426 } 427 428 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); 429 OUT_RING (chan, upper_32_bits(src_addr)); 430 OUT_RING (chan, lower_32_bits(src_addr)); 431 OUT_RING (chan, upper_32_bits(dst_addr)); 432 OUT_RING (chan, lower_32_bits(dst_addr)); 433 OUT_RING (chan, PAGE_SIZE); 434 OUT_RING (chan, PAGE_SIZE); 435 OUT_RING (chan, PAGE_SIZE); 436 OUT_RING (chan, npages); 437 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); 438 OUT_RING (chan, launch_dma); 439 return 0; 440 } 441 442 static int 443 nvc0b5_migrate_clear(struct nouveau_drm *drm, u32 length, 444 enum nouveau_aper dst_aper, u64 dst_addr) 445 { 446 struct nouveau_channel *chan = drm->dmem->migrate.chan; 447 u32 launch_dma = (1 << 10) /* REMAP_ENABLE_TRUE */ | 448 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | 449 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | 450 (1 << 2) /* FLUSH_ENABLE_TRUE. */ | 451 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; 452 u32 remap = (4 << 0) /* DST_X_CONST_A */ | 453 (5 << 4) /* DST_Y_CONST_B */ | 454 (3 << 16) /* COMPONENT_SIZE_FOUR */ | 455 (1 << 24) /* NUM_DST_COMPONENTS_TWO */; 456 int ret; 457 458 ret = RING_SPACE(chan, 12); 459 if (ret) 460 return ret; 461 462 switch (dst_aper) { 463 case NOUVEAU_APER_VRAM: 464 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); 465 break; 466 case NOUVEAU_APER_HOST: 467 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); 468 break; 469 default: 470 return -EINVAL; 471 } 472 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ 473 474 BEGIN_NVC0(chan, NvSubCopy, 0x0700, 3); 475 OUT_RING(chan, 0); 476 OUT_RING(chan, 0); 477 OUT_RING(chan, remap); 478 BEGIN_NVC0(chan, NvSubCopy, 0x0408, 2); 479 OUT_RING(chan, upper_32_bits(dst_addr)); 480 OUT_RING(chan, lower_32_bits(dst_addr)); 481 BEGIN_NVC0(chan, NvSubCopy, 0x0418, 1); 482 OUT_RING(chan, length >> 3); 483 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); 484 OUT_RING(chan, launch_dma); 485 return 0; 486 } 487 488 static int 489 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 490 { 491 switch (drm->ttm.copy.oclass) { 492 case PASCAL_DMA_COPY_A: 493 case PASCAL_DMA_COPY_B: 494 case VOLTA_DMA_COPY_A: 495 case TURING_DMA_COPY_A: 496 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 497 drm->dmem->migrate.clear_func = nvc0b5_migrate_clear; 498 drm->dmem->migrate.chan = drm->ttm.chan; 499 return 0; 500 default: 501 break; 502 } 503 return -ENODEV; 504 } 505 506 void 507 nouveau_dmem_init(struct nouveau_drm *drm) 508 { 509 int ret; 510 511 /* This only make sense on PASCAL or newer */ 512 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 513 return; 514 515 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 516 return; 517 518 drm->dmem->drm = drm; 519 mutex_init(&drm->dmem->mutex); 520 INIT_LIST_HEAD(&drm->dmem->chunks); 521 mutex_init(&drm->dmem->mutex); 522 spin_lock_init(&drm->dmem->lock); 523 524 /* Initialize migration dma helpers before registering memory */ 525 ret = nouveau_dmem_migrate_init(drm); 526 if (ret) { 527 kfree(drm->dmem); 528 drm->dmem = NULL; 529 } 530 } 531 532 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, 533 unsigned long src, dma_addr_t *dma_addr, u64 *pfn) 534 { 535 struct device *dev = drm->dev->dev; 536 struct page *dpage, *spage; 537 unsigned long paddr; 538 539 spage = migrate_pfn_to_page(src); 540 if (!(src & MIGRATE_PFN_MIGRATE)) 541 goto out; 542 543 dpage = nouveau_dmem_page_alloc_locked(drm); 544 if (!dpage) 545 goto out; 546 547 paddr = nouveau_dmem_page_addr(dpage); 548 if (spage) { 549 *dma_addr = dma_map_page(dev, spage, 0, page_size(spage), 550 DMA_BIDIRECTIONAL); 551 if (dma_mapping_error(dev, *dma_addr)) 552 goto out_free_page; 553 if (drm->dmem->migrate.copy_func(drm, page_size(spage), 554 NOUVEAU_APER_VRAM, paddr, NOUVEAU_APER_HOST, *dma_addr)) 555 goto out_dma_unmap; 556 } else { 557 *dma_addr = DMA_MAPPING_ERROR; 558 if (drm->dmem->migrate.clear_func(drm, page_size(dpage), 559 NOUVEAU_APER_VRAM, paddr)) 560 goto out_free_page; 561 } 562 563 *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM | 564 ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT); 565 if (src & MIGRATE_PFN_WRITE) 566 *pfn |= NVIF_VMM_PFNMAP_V0_W; 567 return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 568 569 out_dma_unmap: 570 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 571 out_free_page: 572 nouveau_dmem_page_free_locked(drm, dpage); 573 out: 574 *pfn = NVIF_VMM_PFNMAP_V0_NONE; 575 return 0; 576 } 577 578 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, 579 struct nouveau_svmm *svmm, struct migrate_vma *args, 580 dma_addr_t *dma_addrs, u64 *pfns) 581 { 582 struct nouveau_fence *fence; 583 unsigned long addr = args->start, nr_dma = 0, i; 584 585 for (i = 0; addr < args->end; i++) { 586 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i], 587 dma_addrs + nr_dma, pfns + i); 588 if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma])) 589 nr_dma++; 590 addr += PAGE_SIZE; 591 } 592 593 nouveau_fence_new(drm->dmem->migrate.chan, false, &fence); 594 migrate_vma_pages(args); 595 nouveau_dmem_fence_done(&fence); 596 nouveau_pfns_map(svmm, args->vma->vm_mm, args->start, pfns, i); 597 598 while (nr_dma--) { 599 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE, 600 DMA_BIDIRECTIONAL); 601 } 602 migrate_vma_finalize(args); 603 } 604 605 int 606 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 607 struct nouveau_svmm *svmm, 608 struct vm_area_struct *vma, 609 unsigned long start, 610 unsigned long end) 611 { 612 unsigned long npages = (end - start) >> PAGE_SHIFT; 613 unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages); 614 dma_addr_t *dma_addrs; 615 struct migrate_vma args = { 616 .vma = vma, 617 .start = start, 618 }; 619 unsigned long i; 620 u64 *pfns; 621 int ret = -ENOMEM; 622 623 if (drm->dmem == NULL) 624 return -ENODEV; 625 626 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL); 627 if (!args.src) 628 goto out; 629 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL); 630 if (!args.dst) 631 goto out_free_src; 632 633 dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL); 634 if (!dma_addrs) 635 goto out_free_dst; 636 637 pfns = nouveau_pfns_alloc(max); 638 if (!pfns) 639 goto out_free_dma; 640 641 for (i = 0; i < npages; i += max) { 642 args.end = start + (max << PAGE_SHIFT); 643 ret = migrate_vma_setup(&args); 644 if (ret) 645 goto out_free_pfns; 646 647 if (args.cpages) 648 nouveau_dmem_migrate_chunk(drm, svmm, &args, dma_addrs, 649 pfns); 650 args.start = args.end; 651 } 652 653 ret = 0; 654 out_free_pfns: 655 nouveau_pfns_free(pfns); 656 out_free_dma: 657 kfree(dma_addrs); 658 out_free_dst: 659 kfree(args.dst); 660 out_free_src: 661 kfree(args.src); 662 out: 663 return ret; 664 } 665