1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 }; 96 97 /* 98 * ZONE_DEVICE pages for migration and simulating device memory. 99 */ 100 struct dmirror_chunk { 101 struct dev_pagemap pagemap; 102 struct dmirror_device *mdevice; 103 bool remove; 104 }; 105 106 /* 107 * Per device data. 108 */ 109 struct dmirror_device { 110 struct cdev cdevice; 111 unsigned int zone_device_type; 112 struct device device; 113 114 unsigned int devmem_capacity; 115 unsigned int devmem_count; 116 struct dmirror_chunk **devmem_chunks; 117 struct mutex devmem_lock; /* protects the above */ 118 119 unsigned long calloc; 120 unsigned long cfree; 121 struct page *free_pages; 122 spinlock_t lock; /* protects the above */ 123 }; 124 125 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 126 127 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 128 unsigned long addr, 129 unsigned long size) 130 { 131 bounce->addr = addr; 132 bounce->size = size; 133 bounce->cpages = 0; 134 bounce->ptr = vmalloc(size); 135 if (!bounce->ptr) 136 return -ENOMEM; 137 return 0; 138 } 139 140 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 141 { 142 return (mdevice->zone_device_type == 143 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? true : false; 144 } 145 146 static enum migrate_vma_direction 147 dmirror_select_device(struct dmirror *dmirror) 148 { 149 return (dmirror->mdevice->zone_device_type == 150 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 151 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 152 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 153 } 154 155 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 156 { 157 vfree(bounce->ptr); 158 } 159 160 static int dmirror_fops_open(struct inode *inode, struct file *filp) 161 { 162 struct cdev *cdev = inode->i_cdev; 163 struct dmirror *dmirror; 164 int ret; 165 166 /* Mirror this process address space */ 167 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); 168 if (dmirror == NULL) 169 return -ENOMEM; 170 171 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 172 mutex_init(&dmirror->mutex); 173 xa_init(&dmirror->pt); 174 175 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 176 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 177 if (ret) { 178 kfree(dmirror); 179 return ret; 180 } 181 182 filp->private_data = dmirror; 183 return 0; 184 } 185 186 static int dmirror_fops_release(struct inode *inode, struct file *filp) 187 { 188 struct dmirror *dmirror = filp->private_data; 189 190 mmu_interval_notifier_remove(&dmirror->notifier); 191 xa_destroy(&dmirror->pt); 192 kfree(dmirror); 193 return 0; 194 } 195 196 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) 197 { 198 return container_of(page->pgmap, struct dmirror_chunk, pagemap); 199 } 200 201 static struct dmirror_device *dmirror_page_to_device(struct page *page) 202 203 { 204 return dmirror_page_to_chunk(page)->mdevice; 205 } 206 207 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 208 { 209 unsigned long *pfns = range->hmm_pfns; 210 unsigned long pfn; 211 212 for (pfn = (range->start >> PAGE_SHIFT); 213 pfn < (range->end >> PAGE_SHIFT); 214 pfn++, pfns++) { 215 struct page *page; 216 void *entry; 217 218 /* 219 * Since we asked for hmm_range_fault() to populate pages, 220 * it shouldn't return an error entry on success. 221 */ 222 WARN_ON(*pfns & HMM_PFN_ERROR); 223 WARN_ON(!(*pfns & HMM_PFN_VALID)); 224 225 page = hmm_pfn_to_page(*pfns); 226 WARN_ON(!page); 227 228 entry = page; 229 if (*pfns & HMM_PFN_WRITE) 230 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 231 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 232 return -EFAULT; 233 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 234 if (xa_is_err(entry)) 235 return xa_err(entry); 236 } 237 238 return 0; 239 } 240 241 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 242 unsigned long end) 243 { 244 unsigned long pfn; 245 void *entry; 246 247 /* 248 * The XArray doesn't hold references to pages since it relies on 249 * the mmu notifier to clear page pointers when they become stale. 250 * Therefore, it is OK to just clear the entry. 251 */ 252 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 253 end >> PAGE_SHIFT) 254 xa_erase(&dmirror->pt, pfn); 255 } 256 257 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 258 const struct mmu_notifier_range *range, 259 unsigned long cur_seq) 260 { 261 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 262 263 /* 264 * Ignore invalidation callbacks for device private pages since 265 * the invalidation is handled as part of the migration process. 266 */ 267 if (range->event == MMU_NOTIFY_MIGRATE && 268 range->owner == dmirror->mdevice) 269 return true; 270 271 if (mmu_notifier_range_blockable(range)) 272 mutex_lock(&dmirror->mutex); 273 else if (!mutex_trylock(&dmirror->mutex)) 274 return false; 275 276 mmu_interval_set_seq(mni, cur_seq); 277 dmirror_do_update(dmirror, range->start, range->end); 278 279 mutex_unlock(&dmirror->mutex); 280 return true; 281 } 282 283 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 284 .invalidate = dmirror_interval_invalidate, 285 }; 286 287 static int dmirror_range_fault(struct dmirror *dmirror, 288 struct hmm_range *range) 289 { 290 struct mm_struct *mm = dmirror->notifier.mm; 291 unsigned long timeout = 292 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 293 int ret; 294 295 while (true) { 296 if (time_after(jiffies, timeout)) { 297 ret = -EBUSY; 298 goto out; 299 } 300 301 range->notifier_seq = mmu_interval_read_begin(range->notifier); 302 mmap_read_lock(mm); 303 ret = hmm_range_fault(range); 304 mmap_read_unlock(mm); 305 if (ret) { 306 if (ret == -EBUSY) 307 continue; 308 goto out; 309 } 310 311 mutex_lock(&dmirror->mutex); 312 if (mmu_interval_read_retry(range->notifier, 313 range->notifier_seq)) { 314 mutex_unlock(&dmirror->mutex); 315 continue; 316 } 317 break; 318 } 319 320 ret = dmirror_do_fault(dmirror, range); 321 322 mutex_unlock(&dmirror->mutex); 323 out: 324 return ret; 325 } 326 327 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 328 unsigned long end, bool write) 329 { 330 struct mm_struct *mm = dmirror->notifier.mm; 331 unsigned long addr; 332 unsigned long pfns[64]; 333 struct hmm_range range = { 334 .notifier = &dmirror->notifier, 335 .hmm_pfns = pfns, 336 .pfn_flags_mask = 0, 337 .default_flags = 338 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 339 .dev_private_owner = dmirror->mdevice, 340 }; 341 int ret = 0; 342 343 /* Since the mm is for the mirrored process, get a reference first. */ 344 if (!mmget_not_zero(mm)) 345 return 0; 346 347 for (addr = start; addr < end; addr = range.end) { 348 range.start = addr; 349 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 350 351 ret = dmirror_range_fault(dmirror, &range); 352 if (ret) 353 break; 354 } 355 356 mmput(mm); 357 return ret; 358 } 359 360 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 361 unsigned long end, struct dmirror_bounce *bounce) 362 { 363 unsigned long pfn; 364 void *ptr; 365 366 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 367 368 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 369 void *entry; 370 struct page *page; 371 372 entry = xa_load(&dmirror->pt, pfn); 373 page = xa_untag_pointer(entry); 374 if (!page) 375 return -ENOENT; 376 377 memcpy_from_page(ptr, page, 0, PAGE_SIZE); 378 379 ptr += PAGE_SIZE; 380 bounce->cpages++; 381 } 382 383 return 0; 384 } 385 386 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 387 { 388 struct dmirror_bounce bounce; 389 unsigned long start, end; 390 unsigned long size = cmd->npages << PAGE_SHIFT; 391 int ret; 392 393 start = cmd->addr; 394 end = start + size; 395 if (end < start) 396 return -EINVAL; 397 398 ret = dmirror_bounce_init(&bounce, start, size); 399 if (ret) 400 return ret; 401 402 while (1) { 403 mutex_lock(&dmirror->mutex); 404 ret = dmirror_do_read(dmirror, start, end, &bounce); 405 mutex_unlock(&dmirror->mutex); 406 if (ret != -ENOENT) 407 break; 408 409 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 410 ret = dmirror_fault(dmirror, start, end, false); 411 if (ret) 412 break; 413 cmd->faults++; 414 } 415 416 if (ret == 0) { 417 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 418 bounce.size)) 419 ret = -EFAULT; 420 } 421 cmd->cpages = bounce.cpages; 422 dmirror_bounce_fini(&bounce); 423 return ret; 424 } 425 426 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 427 unsigned long end, struct dmirror_bounce *bounce) 428 { 429 unsigned long pfn; 430 void *ptr; 431 432 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 433 434 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 435 void *entry; 436 struct page *page; 437 438 entry = xa_load(&dmirror->pt, pfn); 439 page = xa_untag_pointer(entry); 440 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 441 return -ENOENT; 442 443 memcpy_to_page(page, 0, ptr, PAGE_SIZE); 444 445 ptr += PAGE_SIZE; 446 bounce->cpages++; 447 } 448 449 return 0; 450 } 451 452 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 453 { 454 struct dmirror_bounce bounce; 455 unsigned long start, end; 456 unsigned long size = cmd->npages << PAGE_SHIFT; 457 int ret; 458 459 start = cmd->addr; 460 end = start + size; 461 if (end < start) 462 return -EINVAL; 463 464 ret = dmirror_bounce_init(&bounce, start, size); 465 if (ret) 466 return ret; 467 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 468 bounce.size)) { 469 ret = -EFAULT; 470 goto fini; 471 } 472 473 while (1) { 474 mutex_lock(&dmirror->mutex); 475 ret = dmirror_do_write(dmirror, start, end, &bounce); 476 mutex_unlock(&dmirror->mutex); 477 if (ret != -ENOENT) 478 break; 479 480 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 481 ret = dmirror_fault(dmirror, start, end, true); 482 if (ret) 483 break; 484 cmd->faults++; 485 } 486 487 fini: 488 cmd->cpages = bounce.cpages; 489 dmirror_bounce_fini(&bounce); 490 return ret; 491 } 492 493 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 494 struct page **ppage) 495 { 496 struct dmirror_chunk *devmem; 497 struct resource *res = NULL; 498 unsigned long pfn; 499 unsigned long pfn_first; 500 unsigned long pfn_last; 501 void *ptr; 502 int ret = -ENOMEM; 503 504 devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); 505 if (!devmem) 506 return ret; 507 508 switch (mdevice->zone_device_type) { 509 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 510 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 511 "hmm_dmirror"); 512 if (IS_ERR_OR_NULL(res)) 513 goto err_devmem; 514 devmem->pagemap.range.start = res->start; 515 devmem->pagemap.range.end = res->end; 516 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 517 break; 518 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 519 devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? 520 spm_addr_dev0 : 521 spm_addr_dev1; 522 devmem->pagemap.range.end = devmem->pagemap.range.start + 523 DEVMEM_CHUNK_SIZE - 1; 524 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 525 break; 526 default: 527 ret = -EINVAL; 528 goto err_devmem; 529 } 530 531 devmem->pagemap.nr_range = 1; 532 devmem->pagemap.ops = &dmirror_devmem_ops; 533 devmem->pagemap.owner = mdevice; 534 535 mutex_lock(&mdevice->devmem_lock); 536 537 if (mdevice->devmem_count == mdevice->devmem_capacity) { 538 struct dmirror_chunk **new_chunks; 539 unsigned int new_capacity; 540 541 new_capacity = mdevice->devmem_capacity + 542 DEVMEM_CHUNKS_RESERVE; 543 new_chunks = krealloc(mdevice->devmem_chunks, 544 sizeof(new_chunks[0]) * new_capacity, 545 GFP_KERNEL); 546 if (!new_chunks) 547 goto err_release; 548 mdevice->devmem_capacity = new_capacity; 549 mdevice->devmem_chunks = new_chunks; 550 } 551 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 552 if (IS_ERR_OR_NULL(ptr)) { 553 if (ptr) 554 ret = PTR_ERR(ptr); 555 else 556 ret = -EFAULT; 557 goto err_release; 558 } 559 560 devmem->mdevice = mdevice; 561 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 562 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 563 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 564 565 mutex_unlock(&mdevice->devmem_lock); 566 567 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 568 DEVMEM_CHUNK_SIZE / (1024 * 1024), 569 mdevice->devmem_count, 570 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 571 pfn_first, pfn_last); 572 573 spin_lock(&mdevice->lock); 574 for (pfn = pfn_first; pfn < pfn_last; pfn++) { 575 struct page *page = pfn_to_page(pfn); 576 577 page->zone_device_data = mdevice->free_pages; 578 mdevice->free_pages = page; 579 } 580 if (ppage) { 581 *ppage = mdevice->free_pages; 582 mdevice->free_pages = (*ppage)->zone_device_data; 583 mdevice->calloc++; 584 } 585 spin_unlock(&mdevice->lock); 586 587 return 0; 588 589 err_release: 590 mutex_unlock(&mdevice->devmem_lock); 591 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 592 release_mem_region(devmem->pagemap.range.start, 593 range_len(&devmem->pagemap.range)); 594 err_devmem: 595 kfree(devmem); 596 597 return ret; 598 } 599 600 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) 601 { 602 struct page *dpage = NULL; 603 struct page *rpage = NULL; 604 605 /* 606 * For ZONE_DEVICE private type, this is a fake device so we allocate 607 * real system memory to store our device memory. 608 * For ZONE_DEVICE coherent type we use the actual dpage to store the 609 * data and ignore rpage. 610 */ 611 if (dmirror_is_private_zone(mdevice)) { 612 rpage = alloc_page(GFP_HIGHUSER); 613 if (!rpage) 614 return NULL; 615 } 616 spin_lock(&mdevice->lock); 617 618 if (mdevice->free_pages) { 619 dpage = mdevice->free_pages; 620 mdevice->free_pages = dpage->zone_device_data; 621 mdevice->calloc++; 622 spin_unlock(&mdevice->lock); 623 } else { 624 spin_unlock(&mdevice->lock); 625 if (dmirror_allocate_chunk(mdevice, &dpage)) 626 goto error; 627 } 628 629 zone_device_page_init(dpage); 630 dpage->zone_device_data = rpage; 631 return dpage; 632 633 error: 634 if (rpage) 635 __free_page(rpage); 636 return NULL; 637 } 638 639 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 640 struct dmirror *dmirror) 641 { 642 struct dmirror_device *mdevice = dmirror->mdevice; 643 const unsigned long *src = args->src; 644 unsigned long *dst = args->dst; 645 unsigned long addr; 646 647 for (addr = args->start; addr < args->end; addr += PAGE_SIZE, 648 src++, dst++) { 649 struct page *spage; 650 struct page *dpage; 651 struct page *rpage; 652 653 if (!(*src & MIGRATE_PFN_MIGRATE)) 654 continue; 655 656 /* 657 * Note that spage might be NULL which is OK since it is an 658 * unallocated pte_none() or read-only zero page. 659 */ 660 spage = migrate_pfn_to_page(*src); 661 if (WARN(spage && is_zone_device_page(spage), 662 "page already in device spage pfn: 0x%lx\n", 663 page_to_pfn(spage))) 664 continue; 665 666 dpage = dmirror_devmem_alloc_page(mdevice); 667 if (!dpage) 668 continue; 669 670 rpage = BACKING_PAGE(dpage); 671 if (spage) 672 copy_highpage(rpage, spage); 673 else 674 clear_highpage(rpage); 675 676 /* 677 * Normally, a device would use the page->zone_device_data to 678 * point to the mirror but here we use it to hold the page for 679 * the simulated device memory and that page holds the pointer 680 * to the mirror. 681 */ 682 rpage->zone_device_data = dmirror; 683 684 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 685 page_to_pfn(spage), page_to_pfn(dpage)); 686 *dst = migrate_pfn(page_to_pfn(dpage)); 687 if ((*src & MIGRATE_PFN_WRITE) || 688 (!spage && args->vma->vm_flags & VM_WRITE)) 689 *dst |= MIGRATE_PFN_WRITE; 690 } 691 } 692 693 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 694 unsigned long end) 695 { 696 unsigned long pfn; 697 698 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 699 void *entry; 700 701 entry = xa_load(&dmirror->pt, pfn); 702 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 703 return -EPERM; 704 } 705 706 return 0; 707 } 708 709 static int dmirror_atomic_map(unsigned long start, unsigned long end, 710 struct page **pages, struct dmirror *dmirror) 711 { 712 unsigned long pfn, mapped = 0; 713 int i; 714 715 /* Map the migrated pages into the device's page tables. */ 716 mutex_lock(&dmirror->mutex); 717 718 for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) { 719 void *entry; 720 721 if (!pages[i]) 722 continue; 723 724 entry = pages[i]; 725 entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); 726 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 727 if (xa_is_err(entry)) { 728 mutex_unlock(&dmirror->mutex); 729 return xa_err(entry); 730 } 731 732 mapped++; 733 } 734 735 mutex_unlock(&dmirror->mutex); 736 return mapped; 737 } 738 739 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 740 struct dmirror *dmirror) 741 { 742 unsigned long start = args->start; 743 unsigned long end = args->end; 744 const unsigned long *src = args->src; 745 const unsigned long *dst = args->dst; 746 unsigned long pfn; 747 748 /* Map the migrated pages into the device's page tables. */ 749 mutex_lock(&dmirror->mutex); 750 751 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, 752 src++, dst++) { 753 struct page *dpage; 754 void *entry; 755 756 if (!(*src & MIGRATE_PFN_MIGRATE)) 757 continue; 758 759 dpage = migrate_pfn_to_page(*dst); 760 if (!dpage) 761 continue; 762 763 entry = BACKING_PAGE(dpage); 764 if (*dst & MIGRATE_PFN_WRITE) 765 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 766 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 767 if (xa_is_err(entry)) { 768 mutex_unlock(&dmirror->mutex); 769 return xa_err(entry); 770 } 771 } 772 773 mutex_unlock(&dmirror->mutex); 774 return 0; 775 } 776 777 static int dmirror_exclusive(struct dmirror *dmirror, 778 struct hmm_dmirror_cmd *cmd) 779 { 780 unsigned long start, end, addr; 781 unsigned long size = cmd->npages << PAGE_SHIFT; 782 struct mm_struct *mm = dmirror->notifier.mm; 783 struct dmirror_bounce bounce; 784 int ret = 0; 785 786 start = cmd->addr; 787 end = start + size; 788 if (end < start) 789 return -EINVAL; 790 791 /* Since the mm is for the mirrored process, get a reference first. */ 792 if (!mmget_not_zero(mm)) 793 return -EINVAL; 794 795 mmap_read_lock(mm); 796 for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { 797 struct folio *folio; 798 struct page *page; 799 800 page = make_device_exclusive(mm, addr, NULL, &folio); 801 if (IS_ERR(page)) { 802 ret = PTR_ERR(page); 803 break; 804 } 805 806 ret = dmirror_atomic_map(addr, addr + PAGE_SIZE, &page, dmirror); 807 ret = ret == 1 ? 0 : -EBUSY; 808 folio_unlock(folio); 809 folio_put(folio); 810 } 811 mmap_read_unlock(mm); 812 mmput(mm); 813 814 if (ret) 815 return ret; 816 817 /* Return the migrated data for verification. */ 818 ret = dmirror_bounce_init(&bounce, start, size); 819 if (ret) 820 return ret; 821 mutex_lock(&dmirror->mutex); 822 ret = dmirror_do_read(dmirror, start, end, &bounce); 823 mutex_unlock(&dmirror->mutex); 824 if (ret == 0) { 825 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 826 bounce.size)) 827 ret = -EFAULT; 828 } 829 830 cmd->cpages = bounce.cpages; 831 dmirror_bounce_fini(&bounce); 832 return ret; 833 } 834 835 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 836 struct dmirror *dmirror) 837 { 838 const unsigned long *src = args->src; 839 unsigned long *dst = args->dst; 840 unsigned long start = args->start; 841 unsigned long end = args->end; 842 unsigned long addr; 843 844 for (addr = start; addr < end; addr += PAGE_SIZE, 845 src++, dst++) { 846 struct page *dpage, *spage; 847 848 spage = migrate_pfn_to_page(*src); 849 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) 850 continue; 851 852 if (WARN_ON(!is_device_private_page(spage) && 853 !is_device_coherent_page(spage))) 854 continue; 855 spage = BACKING_PAGE(spage); 856 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 857 if (!dpage) 858 continue; 859 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", 860 page_to_pfn(spage), page_to_pfn(dpage)); 861 862 lock_page(dpage); 863 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 864 copy_highpage(dpage, spage); 865 *dst = migrate_pfn(page_to_pfn(dpage)); 866 if (*src & MIGRATE_PFN_WRITE) 867 *dst |= MIGRATE_PFN_WRITE; 868 } 869 return 0; 870 } 871 872 static unsigned long 873 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 874 { 875 unsigned long cpages = 0; 876 unsigned long i; 877 878 for (i = 0; i < migrate->npages; i++) { 879 if (migrate->src[i] & MIGRATE_PFN_VALID && 880 migrate->src[i] & MIGRATE_PFN_MIGRATE) 881 cpages++; 882 } 883 return cpages; 884 } 885 886 static int dmirror_migrate_to_system(struct dmirror *dmirror, 887 struct hmm_dmirror_cmd *cmd) 888 { 889 unsigned long start, end, addr; 890 unsigned long size = cmd->npages << PAGE_SHIFT; 891 struct mm_struct *mm = dmirror->notifier.mm; 892 struct vm_area_struct *vma; 893 unsigned long src_pfns[64] = { 0 }; 894 unsigned long dst_pfns[64] = { 0 }; 895 struct migrate_vma args = { 0 }; 896 unsigned long next; 897 int ret; 898 899 start = cmd->addr; 900 end = start + size; 901 if (end < start) 902 return -EINVAL; 903 904 /* Since the mm is for the mirrored process, get a reference first. */ 905 if (!mmget_not_zero(mm)) 906 return -EINVAL; 907 908 cmd->cpages = 0; 909 mmap_read_lock(mm); 910 for (addr = start; addr < end; addr = next) { 911 vma = vma_lookup(mm, addr); 912 if (!vma || !(vma->vm_flags & VM_READ)) { 913 ret = -EINVAL; 914 goto out; 915 } 916 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 917 if (next > vma->vm_end) 918 next = vma->vm_end; 919 920 args.vma = vma; 921 args.src = src_pfns; 922 args.dst = dst_pfns; 923 args.start = addr; 924 args.end = next; 925 args.pgmap_owner = dmirror->mdevice; 926 args.flags = dmirror_select_device(dmirror); 927 928 ret = migrate_vma_setup(&args); 929 if (ret) 930 goto out; 931 932 pr_debug("Migrating from device mem to sys mem\n"); 933 dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 934 935 migrate_vma_pages(&args); 936 cmd->cpages += dmirror_successful_migrated_pages(&args); 937 migrate_vma_finalize(&args); 938 } 939 out: 940 mmap_read_unlock(mm); 941 mmput(mm); 942 943 return ret; 944 } 945 946 static int dmirror_migrate_to_device(struct dmirror *dmirror, 947 struct hmm_dmirror_cmd *cmd) 948 { 949 unsigned long start, end, addr; 950 unsigned long size = cmd->npages << PAGE_SHIFT; 951 struct mm_struct *mm = dmirror->notifier.mm; 952 struct vm_area_struct *vma; 953 unsigned long src_pfns[64] = { 0 }; 954 unsigned long dst_pfns[64] = { 0 }; 955 struct dmirror_bounce bounce; 956 struct migrate_vma args = { 0 }; 957 unsigned long next; 958 int ret; 959 960 start = cmd->addr; 961 end = start + size; 962 if (end < start) 963 return -EINVAL; 964 965 /* Since the mm is for the mirrored process, get a reference first. */ 966 if (!mmget_not_zero(mm)) 967 return -EINVAL; 968 969 mmap_read_lock(mm); 970 for (addr = start; addr < end; addr = next) { 971 vma = vma_lookup(mm, addr); 972 if (!vma || !(vma->vm_flags & VM_READ)) { 973 ret = -EINVAL; 974 goto out; 975 } 976 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 977 if (next > vma->vm_end) 978 next = vma->vm_end; 979 980 args.vma = vma; 981 args.src = src_pfns; 982 args.dst = dst_pfns; 983 args.start = addr; 984 args.end = next; 985 args.pgmap_owner = dmirror->mdevice; 986 args.flags = MIGRATE_VMA_SELECT_SYSTEM; 987 ret = migrate_vma_setup(&args); 988 if (ret) 989 goto out; 990 991 pr_debug("Migrating from sys mem to device mem\n"); 992 dmirror_migrate_alloc_and_copy(&args, dmirror); 993 migrate_vma_pages(&args); 994 dmirror_migrate_finalize_and_map(&args, dmirror); 995 migrate_vma_finalize(&args); 996 } 997 mmap_read_unlock(mm); 998 mmput(mm); 999 1000 /* 1001 * Return the migrated data for verification. 1002 * Only for pages in device zone 1003 */ 1004 ret = dmirror_bounce_init(&bounce, start, size); 1005 if (ret) 1006 return ret; 1007 mutex_lock(&dmirror->mutex); 1008 ret = dmirror_do_read(dmirror, start, end, &bounce); 1009 mutex_unlock(&dmirror->mutex); 1010 if (ret == 0) { 1011 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1012 bounce.size)) 1013 ret = -EFAULT; 1014 } 1015 cmd->cpages = bounce.cpages; 1016 dmirror_bounce_fini(&bounce); 1017 return ret; 1018 1019 out: 1020 mmap_read_unlock(mm); 1021 mmput(mm); 1022 return ret; 1023 } 1024 1025 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1026 unsigned char *perm, unsigned long entry) 1027 { 1028 struct page *page; 1029 1030 if (entry & HMM_PFN_ERROR) { 1031 *perm = HMM_DMIRROR_PROT_ERROR; 1032 return; 1033 } 1034 if (!(entry & HMM_PFN_VALID)) { 1035 *perm = HMM_DMIRROR_PROT_NONE; 1036 return; 1037 } 1038 1039 page = hmm_pfn_to_page(entry); 1040 if (is_device_private_page(page)) { 1041 /* Is the page migrated to this device or some other? */ 1042 if (dmirror->mdevice == dmirror_page_to_device(page)) 1043 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1044 else 1045 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1046 } else if (is_device_coherent_page(page)) { 1047 /* Is the page migrated to this device or some other? */ 1048 if (dmirror->mdevice == dmirror_page_to_device(page)) 1049 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1050 else 1051 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1052 } else if (is_zero_pfn(page_to_pfn(page))) 1053 *perm = HMM_DMIRROR_PROT_ZERO; 1054 else 1055 *perm = HMM_DMIRROR_PROT_NONE; 1056 if (entry & HMM_PFN_WRITE) 1057 *perm |= HMM_DMIRROR_PROT_WRITE; 1058 else 1059 *perm |= HMM_DMIRROR_PROT_READ; 1060 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1061 *perm |= HMM_DMIRROR_PROT_PMD; 1062 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1063 *perm |= HMM_DMIRROR_PROT_PUD; 1064 } 1065 1066 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1067 const struct mmu_notifier_range *range, 1068 unsigned long cur_seq) 1069 { 1070 struct dmirror_interval *dmi = 1071 container_of(mni, struct dmirror_interval, notifier); 1072 struct dmirror *dmirror = dmi->dmirror; 1073 1074 if (mmu_notifier_range_blockable(range)) 1075 mutex_lock(&dmirror->mutex); 1076 else if (!mutex_trylock(&dmirror->mutex)) 1077 return false; 1078 1079 /* 1080 * Snapshots only need to set the sequence number since any 1081 * invalidation in the interval invalidates the whole snapshot. 1082 */ 1083 mmu_interval_set_seq(mni, cur_seq); 1084 1085 mutex_unlock(&dmirror->mutex); 1086 return true; 1087 } 1088 1089 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1090 .invalidate = dmirror_snapshot_invalidate, 1091 }; 1092 1093 static int dmirror_range_snapshot(struct dmirror *dmirror, 1094 struct hmm_range *range, 1095 unsigned char *perm) 1096 { 1097 struct mm_struct *mm = dmirror->notifier.mm; 1098 struct dmirror_interval notifier; 1099 unsigned long timeout = 1100 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1101 unsigned long i; 1102 unsigned long n; 1103 int ret = 0; 1104 1105 notifier.dmirror = dmirror; 1106 range->notifier = ¬ifier.notifier; 1107 1108 ret = mmu_interval_notifier_insert(range->notifier, mm, 1109 range->start, range->end - range->start, 1110 &dmirror_mrn_ops); 1111 if (ret) 1112 return ret; 1113 1114 while (true) { 1115 if (time_after(jiffies, timeout)) { 1116 ret = -EBUSY; 1117 goto out; 1118 } 1119 1120 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1121 1122 mmap_read_lock(mm); 1123 ret = hmm_range_fault(range); 1124 mmap_read_unlock(mm); 1125 if (ret) { 1126 if (ret == -EBUSY) 1127 continue; 1128 goto out; 1129 } 1130 1131 mutex_lock(&dmirror->mutex); 1132 if (mmu_interval_read_retry(range->notifier, 1133 range->notifier_seq)) { 1134 mutex_unlock(&dmirror->mutex); 1135 continue; 1136 } 1137 break; 1138 } 1139 1140 n = (range->end - range->start) >> PAGE_SHIFT; 1141 for (i = 0; i < n; i++) 1142 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1143 1144 mutex_unlock(&dmirror->mutex); 1145 out: 1146 mmu_interval_notifier_remove(range->notifier); 1147 return ret; 1148 } 1149 1150 static int dmirror_snapshot(struct dmirror *dmirror, 1151 struct hmm_dmirror_cmd *cmd) 1152 { 1153 struct mm_struct *mm = dmirror->notifier.mm; 1154 unsigned long start, end; 1155 unsigned long size = cmd->npages << PAGE_SHIFT; 1156 unsigned long addr; 1157 unsigned long next; 1158 unsigned long pfns[64]; 1159 unsigned char perm[64]; 1160 char __user *uptr; 1161 struct hmm_range range = { 1162 .hmm_pfns = pfns, 1163 .dev_private_owner = dmirror->mdevice, 1164 }; 1165 int ret = 0; 1166 1167 start = cmd->addr; 1168 end = start + size; 1169 if (end < start) 1170 return -EINVAL; 1171 1172 /* Since the mm is for the mirrored process, get a reference first. */ 1173 if (!mmget_not_zero(mm)) 1174 return -EINVAL; 1175 1176 /* 1177 * Register a temporary notifier to detect invalidations even if it 1178 * overlaps with other mmu_interval_notifiers. 1179 */ 1180 uptr = u64_to_user_ptr(cmd->ptr); 1181 for (addr = start; addr < end; addr = next) { 1182 unsigned long n; 1183 1184 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1185 range.start = addr; 1186 range.end = next; 1187 1188 ret = dmirror_range_snapshot(dmirror, &range, perm); 1189 if (ret) 1190 break; 1191 1192 n = (range.end - range.start) >> PAGE_SHIFT; 1193 if (copy_to_user(uptr, perm, n)) { 1194 ret = -EFAULT; 1195 break; 1196 } 1197 1198 cmd->cpages += n; 1199 uptr += n; 1200 } 1201 mmput(mm); 1202 1203 return ret; 1204 } 1205 1206 static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) 1207 { 1208 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; 1209 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; 1210 unsigned long npages = end_pfn - start_pfn + 1; 1211 unsigned long i; 1212 unsigned long *src_pfns; 1213 unsigned long *dst_pfns; 1214 1215 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 1216 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 1217 1218 migrate_device_range(src_pfns, start_pfn, npages); 1219 for (i = 0; i < npages; i++) { 1220 struct page *dpage, *spage; 1221 1222 spage = migrate_pfn_to_page(src_pfns[i]); 1223 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 1224 continue; 1225 1226 if (WARN_ON(!is_device_private_page(spage) && 1227 !is_device_coherent_page(spage))) 1228 continue; 1229 spage = BACKING_PAGE(spage); 1230 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); 1231 lock_page(dpage); 1232 copy_highpage(dpage, spage); 1233 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 1234 if (src_pfns[i] & MIGRATE_PFN_WRITE) 1235 dst_pfns[i] |= MIGRATE_PFN_WRITE; 1236 } 1237 migrate_device_pages(src_pfns, dst_pfns, npages); 1238 migrate_device_finalize(src_pfns, dst_pfns, npages); 1239 kvfree(src_pfns); 1240 kvfree(dst_pfns); 1241 } 1242 1243 /* Removes free pages from the free list so they can't be re-allocated */ 1244 static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) 1245 { 1246 struct dmirror_device *mdevice = devmem->mdevice; 1247 struct page *page; 1248 1249 for (page = mdevice->free_pages; page; page = page->zone_device_data) 1250 if (dmirror_page_to_chunk(page) == devmem) 1251 mdevice->free_pages = page->zone_device_data; 1252 } 1253 1254 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) 1255 { 1256 unsigned int i; 1257 1258 mutex_lock(&mdevice->devmem_lock); 1259 if (mdevice->devmem_chunks) { 1260 for (i = 0; i < mdevice->devmem_count; i++) { 1261 struct dmirror_chunk *devmem = 1262 mdevice->devmem_chunks[i]; 1263 1264 spin_lock(&mdevice->lock); 1265 devmem->remove = true; 1266 dmirror_remove_free_pages(devmem); 1267 spin_unlock(&mdevice->lock); 1268 1269 dmirror_device_evict_chunk(devmem); 1270 memunmap_pages(&devmem->pagemap); 1271 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1272 release_mem_region(devmem->pagemap.range.start, 1273 range_len(&devmem->pagemap.range)); 1274 kfree(devmem); 1275 } 1276 mdevice->devmem_count = 0; 1277 mdevice->devmem_capacity = 0; 1278 mdevice->free_pages = NULL; 1279 kfree(mdevice->devmem_chunks); 1280 mdevice->devmem_chunks = NULL; 1281 } 1282 mutex_unlock(&mdevice->devmem_lock); 1283 } 1284 1285 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1286 unsigned int command, 1287 unsigned long arg) 1288 { 1289 void __user *uarg = (void __user *)arg; 1290 struct hmm_dmirror_cmd cmd; 1291 struct dmirror *dmirror; 1292 int ret; 1293 1294 dmirror = filp->private_data; 1295 if (!dmirror) 1296 return -EINVAL; 1297 1298 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1299 return -EFAULT; 1300 1301 if (cmd.addr & ~PAGE_MASK) 1302 return -EINVAL; 1303 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1304 return -EINVAL; 1305 1306 cmd.cpages = 0; 1307 cmd.faults = 0; 1308 1309 switch (command) { 1310 case HMM_DMIRROR_READ: 1311 ret = dmirror_read(dmirror, &cmd); 1312 break; 1313 1314 case HMM_DMIRROR_WRITE: 1315 ret = dmirror_write(dmirror, &cmd); 1316 break; 1317 1318 case HMM_DMIRROR_MIGRATE_TO_DEV: 1319 ret = dmirror_migrate_to_device(dmirror, &cmd); 1320 break; 1321 1322 case HMM_DMIRROR_MIGRATE_TO_SYS: 1323 ret = dmirror_migrate_to_system(dmirror, &cmd); 1324 break; 1325 1326 case HMM_DMIRROR_EXCLUSIVE: 1327 ret = dmirror_exclusive(dmirror, &cmd); 1328 break; 1329 1330 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1331 ret = dmirror_check_atomic(dmirror, cmd.addr, 1332 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1333 break; 1334 1335 case HMM_DMIRROR_SNAPSHOT: 1336 ret = dmirror_snapshot(dmirror, &cmd); 1337 break; 1338 1339 case HMM_DMIRROR_RELEASE: 1340 dmirror_device_remove_chunks(dmirror->mdevice); 1341 ret = 0; 1342 break; 1343 1344 default: 1345 return -EINVAL; 1346 } 1347 if (ret) 1348 return ret; 1349 1350 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1351 return -EFAULT; 1352 1353 return 0; 1354 } 1355 1356 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1357 { 1358 unsigned long addr; 1359 1360 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1361 struct page *page; 1362 int ret; 1363 1364 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1365 if (!page) 1366 return -ENOMEM; 1367 1368 ret = vm_insert_page(vma, addr, page); 1369 if (ret) { 1370 __free_page(page); 1371 return ret; 1372 } 1373 put_page(page); 1374 } 1375 1376 return 0; 1377 } 1378 1379 static const struct file_operations dmirror_fops = { 1380 .open = dmirror_fops_open, 1381 .release = dmirror_fops_release, 1382 .mmap = dmirror_fops_mmap, 1383 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1384 .llseek = default_llseek, 1385 .owner = THIS_MODULE, 1386 }; 1387 1388 static void dmirror_devmem_free(struct page *page) 1389 { 1390 struct page *rpage = BACKING_PAGE(page); 1391 struct dmirror_device *mdevice; 1392 1393 if (rpage != page) 1394 __free_page(rpage); 1395 1396 mdevice = dmirror_page_to_device(page); 1397 spin_lock(&mdevice->lock); 1398 1399 /* Return page to our allocator if not freeing the chunk */ 1400 if (!dmirror_page_to_chunk(page)->remove) { 1401 mdevice->cfree++; 1402 page->zone_device_data = mdevice->free_pages; 1403 mdevice->free_pages = page; 1404 } 1405 spin_unlock(&mdevice->lock); 1406 } 1407 1408 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1409 { 1410 struct migrate_vma args = { 0 }; 1411 unsigned long src_pfns = 0; 1412 unsigned long dst_pfns = 0; 1413 struct page *rpage; 1414 struct dmirror *dmirror; 1415 vm_fault_t ret; 1416 1417 /* 1418 * Normally, a device would use the page->zone_device_data to point to 1419 * the mirror but here we use it to hold the page for the simulated 1420 * device memory and that page holds the pointer to the mirror. 1421 */ 1422 rpage = vmf->page->zone_device_data; 1423 dmirror = rpage->zone_device_data; 1424 1425 /* FIXME demonstrate how we can adjust migrate range */ 1426 args.vma = vmf->vma; 1427 args.start = vmf->address; 1428 args.end = args.start + PAGE_SIZE; 1429 args.src = &src_pfns; 1430 args.dst = &dst_pfns; 1431 args.pgmap_owner = dmirror->mdevice; 1432 args.flags = dmirror_select_device(dmirror); 1433 args.fault_page = vmf->page; 1434 1435 if (migrate_vma_setup(&args)) 1436 return VM_FAULT_SIGBUS; 1437 1438 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1439 if (ret) 1440 return ret; 1441 migrate_vma_pages(&args); 1442 /* 1443 * No device finalize step is needed since 1444 * dmirror_devmem_fault_alloc_and_copy() will have already 1445 * invalidated the device page table. 1446 */ 1447 migrate_vma_finalize(&args); 1448 return 0; 1449 } 1450 1451 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1452 .page_free = dmirror_devmem_free, 1453 .migrate_to_ram = dmirror_devmem_fault, 1454 }; 1455 1456 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1457 { 1458 dev_t dev; 1459 int ret; 1460 1461 dev = MKDEV(MAJOR(dmirror_dev), id); 1462 mutex_init(&mdevice->devmem_lock); 1463 spin_lock_init(&mdevice->lock); 1464 1465 cdev_init(&mdevice->cdevice, &dmirror_fops); 1466 mdevice->cdevice.owner = THIS_MODULE; 1467 device_initialize(&mdevice->device); 1468 mdevice->device.devt = dev; 1469 1470 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); 1471 if (ret) 1472 return ret; 1473 1474 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); 1475 if (ret) 1476 return ret; 1477 1478 /* Build a list of free ZONE_DEVICE struct pages */ 1479 return dmirror_allocate_chunk(mdevice, NULL); 1480 } 1481 1482 static void dmirror_device_remove(struct dmirror_device *mdevice) 1483 { 1484 dmirror_device_remove_chunks(mdevice); 1485 cdev_device_del(&mdevice->cdevice, &mdevice->device); 1486 } 1487 1488 static int __init hmm_dmirror_init(void) 1489 { 1490 int ret; 1491 int id = 0; 1492 int ndevices = 0; 1493 1494 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1495 "HMM_DMIRROR"); 1496 if (ret) 1497 goto err_unreg; 1498 1499 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1500 dmirror_devices[ndevices++].zone_device_type = 1501 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1502 dmirror_devices[ndevices++].zone_device_type = 1503 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1504 if (spm_addr_dev0 && spm_addr_dev1) { 1505 dmirror_devices[ndevices++].zone_device_type = 1506 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1507 dmirror_devices[ndevices++].zone_device_type = 1508 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1509 } 1510 for (id = 0; id < ndevices; id++) { 1511 ret = dmirror_device_init(dmirror_devices + id, id); 1512 if (ret) 1513 goto err_chrdev; 1514 } 1515 1516 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1517 return 0; 1518 1519 err_chrdev: 1520 while (--id >= 0) 1521 dmirror_device_remove(dmirror_devices + id); 1522 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1523 err_unreg: 1524 return ret; 1525 } 1526 1527 static void __exit hmm_dmirror_exit(void) 1528 { 1529 int id; 1530 1531 for (id = 0; id < DMIRROR_NDEVICES; id++) 1532 if (dmirror_devices[id].zone_device_type) 1533 dmirror_device_remove(dmirror_devices + id); 1534 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1535 } 1536 1537 module_init(hmm_dmirror_init); 1538 module_exit(hmm_dmirror_exit); 1539 MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module"); 1540 MODULE_LICENSE("GPL"); 1541