1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_dmem.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dma.h" 26 #include "nouveau_mem.h" 27 #include "nouveau_bo.h" 28 #include "nouveau_svm.h" 29 30 #include <nvif/class.h> 31 #include <nvif/object.h> 32 #include <nvif/push906f.h> 33 #include <nvif/if000c.h> 34 #include <nvif/if500b.h> 35 #include <nvif/if900b.h> 36 37 #include <nvhw/class/cla0b5.h> 38 39 #include <linux/sched/mm.h> 40 #include <linux/hmm.h> 41 #include <linux/memremap.h> 42 #include <linux/migrate.h> 43 44 /* 45 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 46 * it in vram while in use. We likely want to overhaul memory management for 47 * nouveau to be more page like (not necessarily with system page size but a 48 * bigger page size) at lowest level and have some shim layer on top that would 49 * provide the same functionality as TTM. 50 */ 51 #define DMEM_CHUNK_SIZE (2UL << 20) 52 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 53 54 enum nouveau_aper { 55 NOUVEAU_APER_VIRT, 56 NOUVEAU_APER_VRAM, 57 NOUVEAU_APER_HOST, 58 }; 59 60 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 61 enum nouveau_aper, u64 dst_addr, 62 enum nouveau_aper, u64 src_addr); 63 typedef int (*nouveau_clear_page_t)(struct nouveau_drm *drm, u32 length, 64 enum nouveau_aper, u64 dst_addr); 65 66 struct nouveau_dmem_chunk { 67 struct list_head list; 68 struct nouveau_bo *bo; 69 struct nouveau_drm *drm; 70 unsigned long callocated; 71 struct dev_pagemap pagemap; 72 }; 73 74 struct nouveau_dmem_migrate { 75 nouveau_migrate_copy_t copy_func; 76 nouveau_clear_page_t clear_func; 77 struct nouveau_channel *chan; 78 }; 79 80 struct nouveau_dmem { 81 struct nouveau_drm *drm; 82 struct nouveau_dmem_migrate migrate; 83 struct list_head chunks; 84 struct mutex mutex; 85 struct page *free_pages; 86 spinlock_t lock; 87 }; 88 89 static struct nouveau_dmem_chunk *nouveau_page_to_chunk(struct page *page) 90 { 91 return container_of(page_pgmap(page), struct nouveau_dmem_chunk, 92 pagemap); 93 } 94 95 static struct nouveau_drm *page_to_drm(struct page *page) 96 { 97 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 98 99 return chunk->drm; 100 } 101 102 unsigned long nouveau_dmem_page_addr(struct page *page) 103 { 104 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 105 unsigned long off = (page_to_pfn(page) << PAGE_SHIFT) - 106 chunk->pagemap.range.start; 107 108 return chunk->bo->offset + off; 109 } 110 111 static void nouveau_dmem_page_free(struct page *page) 112 { 113 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 114 struct nouveau_dmem *dmem = chunk->drm->dmem; 115 116 spin_lock(&dmem->lock); 117 page->zone_device_data = dmem->free_pages; 118 dmem->free_pages = page; 119 120 WARN_ON(!chunk->callocated); 121 chunk->callocated--; 122 /* 123 * FIXME when chunk->callocated reach 0 we should add the chunk to 124 * a reclaim list so that it can be freed in case of memory pressure. 125 */ 126 spin_unlock(&dmem->lock); 127 } 128 129 static void nouveau_dmem_fence_done(struct nouveau_fence **fence) 130 { 131 if (fence) { 132 nouveau_fence_wait(*fence, true, false); 133 nouveau_fence_unref(fence); 134 } else { 135 /* 136 * FIXME wait for channel to be IDLE before calling finalizing 137 * the hmem object. 138 */ 139 } 140 } 141 142 static int nouveau_dmem_copy_one(struct nouveau_drm *drm, struct page *spage, 143 struct page *dpage, dma_addr_t *dma_addr) 144 { 145 struct device *dev = drm->dev->dev; 146 147 lock_page(dpage); 148 149 *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 150 if (dma_mapping_error(dev, *dma_addr)) 151 return -EIO; 152 153 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr, 154 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) { 155 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 156 return -EIO; 157 } 158 159 return 0; 160 } 161 162 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) 163 { 164 struct nouveau_drm *drm = page_to_drm(vmf->page); 165 struct nouveau_dmem *dmem = drm->dmem; 166 struct nouveau_fence *fence; 167 struct nouveau_svmm *svmm; 168 struct page *spage, *dpage; 169 unsigned long src = 0, dst = 0; 170 dma_addr_t dma_addr = 0; 171 vm_fault_t ret = 0; 172 struct migrate_vma args = { 173 .vma = vmf->vma, 174 .start = vmf->address, 175 .end = vmf->address + PAGE_SIZE, 176 .src = &src, 177 .dst = &dst, 178 .pgmap_owner = drm->dev, 179 .fault_page = vmf->page, 180 .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE, 181 }; 182 183 /* 184 * FIXME what we really want is to find some heuristic to migrate more 185 * than just one page on CPU fault. When such fault happens it is very 186 * likely that more surrounding page will CPU fault too. 187 */ 188 if (migrate_vma_setup(&args) < 0) 189 return VM_FAULT_SIGBUS; 190 if (!args.cpages) 191 return 0; 192 193 spage = migrate_pfn_to_page(src); 194 if (!spage || !(src & MIGRATE_PFN_MIGRATE)) 195 goto done; 196 197 dpage = alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vmf->vma, vmf->address); 198 if (!dpage) 199 goto done; 200 201 dst = migrate_pfn(page_to_pfn(dpage)); 202 203 svmm = spage->zone_device_data; 204 mutex_lock(&svmm->mutex); 205 nouveau_svmm_invalidate(svmm, args.start, args.end); 206 ret = nouveau_dmem_copy_one(drm, spage, dpage, &dma_addr); 207 mutex_unlock(&svmm->mutex); 208 if (ret) { 209 ret = VM_FAULT_SIGBUS; 210 goto done; 211 } 212 213 nouveau_fence_new(&fence, dmem->migrate.chan); 214 migrate_vma_pages(&args); 215 nouveau_dmem_fence_done(&fence); 216 dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 217 done: 218 migrate_vma_finalize(&args); 219 return ret; 220 } 221 222 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { 223 .page_free = nouveau_dmem_page_free, 224 .migrate_to_ram = nouveau_dmem_migrate_to_ram, 225 }; 226 227 static int 228 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage) 229 { 230 struct nouveau_dmem_chunk *chunk; 231 struct resource *res; 232 struct page *page; 233 void *ptr; 234 unsigned long i, pfn_first; 235 int ret; 236 237 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 238 if (chunk == NULL) { 239 ret = -ENOMEM; 240 goto out; 241 } 242 243 /* Allocate unused physical address space for device private pages. */ 244 res = request_free_mem_region(&iomem_resource, DMEM_CHUNK_SIZE, 245 "nouveau_dmem"); 246 if (IS_ERR(res)) { 247 ret = PTR_ERR(res); 248 goto out_free; 249 } 250 251 chunk->drm = drm; 252 chunk->pagemap.type = MEMORY_DEVICE_PRIVATE; 253 chunk->pagemap.range.start = res->start; 254 chunk->pagemap.range.end = res->end; 255 chunk->pagemap.nr_range = 1; 256 chunk->pagemap.ops = &nouveau_dmem_pagemap_ops; 257 chunk->pagemap.owner = drm->dev; 258 259 ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, DMEM_CHUNK_SIZE, 260 &chunk->bo); 261 if (ret) 262 goto out_release; 263 264 ptr = memremap_pages(&chunk->pagemap, numa_node_id()); 265 if (IS_ERR(ptr)) { 266 ret = PTR_ERR(ptr); 267 goto out_bo_free; 268 } 269 270 mutex_lock(&drm->dmem->mutex); 271 list_add(&chunk->list, &drm->dmem->chunks); 272 mutex_unlock(&drm->dmem->mutex); 273 274 pfn_first = chunk->pagemap.range.start >> PAGE_SHIFT; 275 page = pfn_to_page(pfn_first); 276 spin_lock(&drm->dmem->lock); 277 for (i = 0; i < DMEM_CHUNK_NPAGES - 1; ++i, ++page) { 278 page->zone_device_data = drm->dmem->free_pages; 279 drm->dmem->free_pages = page; 280 } 281 *ppage = page; 282 chunk->callocated++; 283 spin_unlock(&drm->dmem->lock); 284 285 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", 286 DMEM_CHUNK_SIZE >> 20); 287 288 return 0; 289 290 out_bo_free: 291 nouveau_bo_unpin_del(&chunk->bo); 292 out_release: 293 release_mem_region(chunk->pagemap.range.start, range_len(&chunk->pagemap.range)); 294 out_free: 295 kfree(chunk); 296 out: 297 return ret; 298 } 299 300 static struct page * 301 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 302 { 303 struct nouveau_dmem_chunk *chunk; 304 struct page *page = NULL; 305 int ret; 306 307 spin_lock(&drm->dmem->lock); 308 if (drm->dmem->free_pages) { 309 page = drm->dmem->free_pages; 310 drm->dmem->free_pages = page->zone_device_data; 311 chunk = nouveau_page_to_chunk(page); 312 chunk->callocated++; 313 spin_unlock(&drm->dmem->lock); 314 } else { 315 spin_unlock(&drm->dmem->lock); 316 ret = nouveau_dmem_chunk_alloc(drm, &page); 317 if (ret) 318 return NULL; 319 } 320 321 zone_device_page_init(page); 322 return page; 323 } 324 325 static void 326 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 327 { 328 unlock_page(page); 329 put_page(page); 330 } 331 332 void 333 nouveau_dmem_resume(struct nouveau_drm *drm) 334 { 335 struct nouveau_dmem_chunk *chunk; 336 int ret; 337 338 if (drm->dmem == NULL) 339 return; 340 341 mutex_lock(&drm->dmem->mutex); 342 list_for_each_entry(chunk, &drm->dmem->chunks, list) { 343 ret = nouveau_bo_pin(chunk->bo, NOUVEAU_GEM_DOMAIN_VRAM, false); 344 /* FIXME handle pin failure */ 345 WARN_ON(ret); 346 } 347 mutex_unlock(&drm->dmem->mutex); 348 } 349 350 void 351 nouveau_dmem_suspend(struct nouveau_drm *drm) 352 { 353 struct nouveau_dmem_chunk *chunk; 354 355 if (drm->dmem == NULL) 356 return; 357 358 mutex_lock(&drm->dmem->mutex); 359 list_for_each_entry(chunk, &drm->dmem->chunks, list) 360 nouveau_bo_unpin(chunk->bo); 361 mutex_unlock(&drm->dmem->mutex); 362 } 363 364 /* 365 * Evict all pages mapping a chunk. 366 */ 367 static void 368 nouveau_dmem_evict_chunk(struct nouveau_dmem_chunk *chunk) 369 { 370 unsigned long i, npages = range_len(&chunk->pagemap.range) >> PAGE_SHIFT; 371 unsigned long *src_pfns, *dst_pfns; 372 dma_addr_t *dma_addrs; 373 struct nouveau_fence *fence; 374 375 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 376 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 377 dma_addrs = kvcalloc(npages, sizeof(*dma_addrs), GFP_KERNEL | __GFP_NOFAIL); 378 379 migrate_device_range(src_pfns, chunk->pagemap.range.start >> PAGE_SHIFT, 380 npages); 381 382 for (i = 0; i < npages; i++) { 383 if (src_pfns[i] & MIGRATE_PFN_MIGRATE) { 384 struct page *dpage; 385 386 /* 387 * _GFP_NOFAIL because the GPU is going away and there 388 * is nothing sensible we can do if we can't copy the 389 * data back. 390 */ 391 dpage = alloc_page(GFP_HIGHUSER | __GFP_NOFAIL); 392 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 393 nouveau_dmem_copy_one(chunk->drm, 394 migrate_pfn_to_page(src_pfns[i]), dpage, 395 &dma_addrs[i]); 396 } 397 } 398 399 nouveau_fence_new(&fence, chunk->drm->dmem->migrate.chan); 400 migrate_device_pages(src_pfns, dst_pfns, npages); 401 nouveau_dmem_fence_done(&fence); 402 migrate_device_finalize(src_pfns, dst_pfns, npages); 403 kvfree(src_pfns); 404 kvfree(dst_pfns); 405 for (i = 0; i < npages; i++) 406 dma_unmap_page(chunk->drm->dev->dev, dma_addrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 407 kvfree(dma_addrs); 408 } 409 410 void 411 nouveau_dmem_fini(struct nouveau_drm *drm) 412 { 413 struct nouveau_dmem_chunk *chunk, *tmp; 414 415 if (drm->dmem == NULL) 416 return; 417 418 mutex_lock(&drm->dmem->mutex); 419 420 list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) { 421 nouveau_dmem_evict_chunk(chunk); 422 nouveau_bo_unpin_del(&chunk->bo); 423 WARN_ON(chunk->callocated); 424 list_del(&chunk->list); 425 memunmap_pages(&chunk->pagemap); 426 release_mem_region(chunk->pagemap.range.start, 427 range_len(&chunk->pagemap.range)); 428 kfree(chunk); 429 } 430 431 mutex_unlock(&drm->dmem->mutex); 432 } 433 434 static int 435 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 436 enum nouveau_aper dst_aper, u64 dst_addr, 437 enum nouveau_aper src_aper, u64 src_addr) 438 { 439 struct nvif_push *push = &drm->dmem->migrate.chan->chan.push; 440 u32 launch_dma = 0; 441 int ret; 442 443 ret = PUSH_WAIT(push, 13); 444 if (ret) 445 return ret; 446 447 if (src_aper != NOUVEAU_APER_VIRT) { 448 switch (src_aper) { 449 case NOUVEAU_APER_VRAM: 450 PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE, 451 NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, LOCAL_FB)); 452 break; 453 case NOUVEAU_APER_HOST: 454 PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE, 455 NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, COHERENT_SYSMEM)); 456 break; 457 default: 458 return -EINVAL; 459 } 460 461 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, SRC_TYPE, PHYSICAL); 462 } 463 464 if (dst_aper != NOUVEAU_APER_VIRT) { 465 switch (dst_aper) { 466 case NOUVEAU_APER_VRAM: 467 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 468 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB)); 469 break; 470 case NOUVEAU_APER_HOST: 471 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 472 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM)); 473 break; 474 default: 475 return -EINVAL; 476 } 477 478 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL); 479 } 480 481 PUSH_MTHD(push, NVA0B5, OFFSET_IN_UPPER, 482 NVVAL(NVA0B5, OFFSET_IN_UPPER, UPPER, upper_32_bits(src_addr)), 483 484 OFFSET_IN_LOWER, lower_32_bits(src_addr), 485 486 OFFSET_OUT_UPPER, 487 NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)), 488 489 OFFSET_OUT_LOWER, lower_32_bits(dst_addr), 490 PITCH_IN, PAGE_SIZE, 491 PITCH_OUT, PAGE_SIZE, 492 LINE_LENGTH_IN, PAGE_SIZE, 493 LINE_COUNT, npages); 494 495 PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma | 496 NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) | 497 NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) | 498 NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) | 499 NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) | 500 NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) | 501 NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) | 502 NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, TRUE) | 503 NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, FALSE) | 504 NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING)); 505 return 0; 506 } 507 508 static int 509 nvc0b5_migrate_clear(struct nouveau_drm *drm, u32 length, 510 enum nouveau_aper dst_aper, u64 dst_addr) 511 { 512 struct nvif_push *push = &drm->dmem->migrate.chan->chan.push; 513 u32 launch_dma = 0; 514 int ret; 515 516 ret = PUSH_WAIT(push, 12); 517 if (ret) 518 return ret; 519 520 switch (dst_aper) { 521 case NOUVEAU_APER_VRAM: 522 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 523 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB)); 524 break; 525 case NOUVEAU_APER_HOST: 526 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 527 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM)); 528 break; 529 default: 530 return -EINVAL; 531 } 532 533 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL); 534 535 PUSH_MTHD(push, NVA0B5, SET_REMAP_CONST_A, 0, 536 SET_REMAP_CONST_B, 0, 537 538 SET_REMAP_COMPONENTS, 539 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_X, CONST_A) | 540 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_Y, CONST_B) | 541 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, COMPONENT_SIZE, FOUR) | 542 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, NUM_DST_COMPONENTS, TWO)); 543 544 PUSH_MTHD(push, NVA0B5, OFFSET_OUT_UPPER, 545 NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)), 546 547 OFFSET_OUT_LOWER, lower_32_bits(dst_addr)); 548 549 PUSH_MTHD(push, NVA0B5, LINE_LENGTH_IN, length >> 3); 550 551 PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma | 552 NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) | 553 NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) | 554 NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) | 555 NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) | 556 NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) | 557 NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) | 558 NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, FALSE) | 559 NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, TRUE) | 560 NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING)); 561 return 0; 562 } 563 564 static int 565 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 566 { 567 switch (drm->ttm.copy.oclass) { 568 case PASCAL_DMA_COPY_A: 569 case PASCAL_DMA_COPY_B: 570 case VOLTA_DMA_COPY_A: 571 case TURING_DMA_COPY_A: 572 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 573 drm->dmem->migrate.clear_func = nvc0b5_migrate_clear; 574 drm->dmem->migrate.chan = drm->ttm.chan; 575 return 0; 576 default: 577 break; 578 } 579 return -ENODEV; 580 } 581 582 void 583 nouveau_dmem_init(struct nouveau_drm *drm) 584 { 585 int ret; 586 587 /* This only make sense on PASCAL or newer */ 588 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 589 return; 590 591 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 592 return; 593 594 drm->dmem->drm = drm; 595 mutex_init(&drm->dmem->mutex); 596 INIT_LIST_HEAD(&drm->dmem->chunks); 597 mutex_init(&drm->dmem->mutex); 598 spin_lock_init(&drm->dmem->lock); 599 600 /* Initialize migration dma helpers before registering memory */ 601 ret = nouveau_dmem_migrate_init(drm); 602 if (ret) { 603 kfree(drm->dmem); 604 drm->dmem = NULL; 605 } 606 } 607 608 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, 609 struct nouveau_svmm *svmm, unsigned long src, 610 dma_addr_t *dma_addr, u64 *pfn) 611 { 612 struct device *dev = drm->dev->dev; 613 struct page *dpage, *spage; 614 unsigned long paddr; 615 616 spage = migrate_pfn_to_page(src); 617 if (!(src & MIGRATE_PFN_MIGRATE)) 618 goto out; 619 620 dpage = nouveau_dmem_page_alloc_locked(drm); 621 if (!dpage) 622 goto out; 623 624 paddr = nouveau_dmem_page_addr(dpage); 625 if (spage) { 626 *dma_addr = dma_map_page(dev, spage, 0, page_size(spage), 627 DMA_BIDIRECTIONAL); 628 if (dma_mapping_error(dev, *dma_addr)) 629 goto out_free_page; 630 if (drm->dmem->migrate.copy_func(drm, 1, 631 NOUVEAU_APER_VRAM, paddr, NOUVEAU_APER_HOST, *dma_addr)) 632 goto out_dma_unmap; 633 } else { 634 *dma_addr = DMA_MAPPING_ERROR; 635 if (drm->dmem->migrate.clear_func(drm, page_size(dpage), 636 NOUVEAU_APER_VRAM, paddr)) 637 goto out_free_page; 638 } 639 640 dpage->zone_device_data = svmm; 641 *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM | 642 ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT); 643 if (src & MIGRATE_PFN_WRITE) 644 *pfn |= NVIF_VMM_PFNMAP_V0_W; 645 return migrate_pfn(page_to_pfn(dpage)); 646 647 out_dma_unmap: 648 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 649 out_free_page: 650 nouveau_dmem_page_free_locked(drm, dpage); 651 out: 652 *pfn = NVIF_VMM_PFNMAP_V0_NONE; 653 return 0; 654 } 655 656 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, 657 struct nouveau_svmm *svmm, struct migrate_vma *args, 658 dma_addr_t *dma_addrs, u64 *pfns) 659 { 660 struct nouveau_fence *fence; 661 unsigned long addr = args->start, nr_dma = 0, i; 662 663 for (i = 0; addr < args->end; i++) { 664 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, svmm, 665 args->src[i], dma_addrs + nr_dma, pfns + i); 666 if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma])) 667 nr_dma++; 668 addr += PAGE_SIZE; 669 } 670 671 nouveau_fence_new(&fence, drm->dmem->migrate.chan); 672 migrate_vma_pages(args); 673 nouveau_dmem_fence_done(&fence); 674 nouveau_pfns_map(svmm, args->vma->vm_mm, args->start, pfns, i); 675 676 while (nr_dma--) { 677 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE, 678 DMA_BIDIRECTIONAL); 679 } 680 migrate_vma_finalize(args); 681 } 682 683 int 684 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 685 struct nouveau_svmm *svmm, 686 struct vm_area_struct *vma, 687 unsigned long start, 688 unsigned long end) 689 { 690 unsigned long npages = (end - start) >> PAGE_SHIFT; 691 unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages); 692 dma_addr_t *dma_addrs; 693 struct migrate_vma args = { 694 .vma = vma, 695 .start = start, 696 .pgmap_owner = drm->dev, 697 .flags = MIGRATE_VMA_SELECT_SYSTEM, 698 }; 699 unsigned long i; 700 u64 *pfns; 701 int ret = -ENOMEM; 702 703 if (drm->dmem == NULL) 704 return -ENODEV; 705 706 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL); 707 if (!args.src) 708 goto out; 709 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL); 710 if (!args.dst) 711 goto out_free_src; 712 713 dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL); 714 if (!dma_addrs) 715 goto out_free_dst; 716 717 pfns = nouveau_pfns_alloc(max); 718 if (!pfns) 719 goto out_free_dma; 720 721 for (i = 0; i < npages; i += max) { 722 if (args.start + (max << PAGE_SHIFT) > end) 723 args.end = end; 724 else 725 args.end = args.start + (max << PAGE_SHIFT); 726 727 ret = migrate_vma_setup(&args); 728 if (ret) 729 goto out_free_pfns; 730 731 if (args.cpages) 732 nouveau_dmem_migrate_chunk(drm, svmm, &args, dma_addrs, 733 pfns); 734 args.start = args.end; 735 } 736 737 ret = 0; 738 out_free_pfns: 739 nouveau_pfns_free(pfns); 740 out_free_dma: 741 kfree(dma_addrs); 742 out_free_dst: 743 kfree(args.dst); 744 out_free_src: 745 kfree(args.src); 746 out: 747 return ret; 748 } 749