1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 __u64 flags; 96 }; 97 98 /* 99 * ZONE_DEVICE pages for migration and simulating device memory. 100 */ 101 struct dmirror_chunk { 102 struct dev_pagemap pagemap; 103 struct dmirror_device *mdevice; 104 bool remove; 105 }; 106 107 /* 108 * Per device data. 109 */ 110 struct dmirror_device { 111 struct cdev cdevice; 112 unsigned int zone_device_type; 113 struct device device; 114 115 unsigned int devmem_capacity; 116 unsigned int devmem_count; 117 struct dmirror_chunk **devmem_chunks; 118 struct mutex devmem_lock; /* protects the above */ 119 120 unsigned long calloc; 121 unsigned long cfree; 122 struct page *free_pages; 123 struct folio *free_folios; 124 spinlock_t lock; /* protects the above */ 125 }; 126 127 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 128 129 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 130 unsigned long addr, 131 unsigned long size) 132 { 133 bounce->addr = addr; 134 bounce->size = size; 135 bounce->cpages = 0; 136 bounce->ptr = vmalloc(size); 137 if (!bounce->ptr) 138 return -ENOMEM; 139 return 0; 140 } 141 142 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 143 { 144 return (mdevice->zone_device_type == 145 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE); 146 } 147 148 static enum migrate_vma_direction 149 dmirror_select_device(struct dmirror *dmirror) 150 { 151 return (dmirror->mdevice->zone_device_type == 152 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 153 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 154 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 155 } 156 157 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 158 { 159 vfree(bounce->ptr); 160 } 161 162 static int dmirror_fops_open(struct inode *inode, struct file *filp) 163 { 164 struct cdev *cdev = inode->i_cdev; 165 struct dmirror *dmirror; 166 int ret; 167 168 /* Mirror this process address space */ 169 dmirror = kzalloc_obj(*dmirror); 170 if (dmirror == NULL) 171 return -ENOMEM; 172 173 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 174 mutex_init(&dmirror->mutex); 175 xa_init(&dmirror->pt); 176 177 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 178 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 179 if (ret) { 180 kfree(dmirror); 181 return ret; 182 } 183 184 filp->private_data = dmirror; 185 return 0; 186 } 187 188 static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) 189 { 190 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; 191 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; 192 unsigned long npages = end_pfn - start_pfn + 1; 193 unsigned long i; 194 unsigned long *src_pfns; 195 unsigned long *dst_pfns; 196 unsigned int order = 0; 197 198 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 199 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 200 201 migrate_device_range(src_pfns, start_pfn, npages); 202 for (i = 0; i < npages; i++) { 203 struct page *dpage, *spage; 204 205 spage = migrate_pfn_to_page(src_pfns[i]); 206 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 207 continue; 208 209 if (WARN_ON(!is_device_private_page(spage) && 210 !is_device_coherent_page(spage))) 211 continue; 212 213 order = folio_order(page_folio(spage)); 214 spage = BACKING_PAGE(spage); 215 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { 216 dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, 217 order), 0); 218 } else { 219 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); 220 order = 0; 221 } 222 223 /* TODO Support splitting here */ 224 lock_page(dpage); 225 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 226 if (src_pfns[i] & MIGRATE_PFN_WRITE) 227 dst_pfns[i] |= MIGRATE_PFN_WRITE; 228 if (order) 229 dst_pfns[i] |= MIGRATE_PFN_COMPOUND; 230 folio_copy(page_folio(dpage), page_folio(spage)); 231 } 232 migrate_device_pages(src_pfns, dst_pfns, npages); 233 migrate_device_finalize(src_pfns, dst_pfns, npages); 234 kvfree(src_pfns); 235 kvfree(dst_pfns); 236 } 237 238 static int dmirror_fops_release(struct inode *inode, struct file *filp) 239 { 240 struct dmirror *dmirror = filp->private_data; 241 struct dmirror_device *mdevice = dmirror->mdevice; 242 int i; 243 244 mmu_interval_notifier_remove(&dmirror->notifier); 245 246 if (mdevice->devmem_chunks) { 247 for (i = 0; i < mdevice->devmem_count; i++) { 248 struct dmirror_chunk *devmem = 249 mdevice->devmem_chunks[i]; 250 251 dmirror_device_evict_chunk(devmem); 252 } 253 } 254 255 xa_destroy(&dmirror->pt); 256 kfree(dmirror); 257 return 0; 258 } 259 260 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) 261 { 262 return container_of(page_pgmap(page), struct dmirror_chunk, 263 pagemap); 264 } 265 266 static struct dmirror_device *dmirror_page_to_device(struct page *page) 267 268 { 269 return dmirror_page_to_chunk(page)->mdevice; 270 } 271 272 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 273 { 274 unsigned long *pfns = range->hmm_pfns; 275 unsigned long pfn; 276 277 for (pfn = (range->start >> PAGE_SHIFT); 278 pfn < (range->end >> PAGE_SHIFT); 279 pfn++, pfns++) { 280 struct page *page; 281 void *entry; 282 283 /* 284 * Since we asked for hmm_range_fault() to populate pages, 285 * it shouldn't return an error entry on success. 286 */ 287 WARN_ON(*pfns & HMM_PFN_ERROR); 288 WARN_ON(!(*pfns & HMM_PFN_VALID)); 289 290 page = hmm_pfn_to_page(*pfns); 291 WARN_ON(!page); 292 293 entry = page; 294 if (*pfns & HMM_PFN_WRITE) 295 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 296 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 297 return -EFAULT; 298 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 299 if (xa_is_err(entry)) 300 return xa_err(entry); 301 } 302 303 return 0; 304 } 305 306 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 307 unsigned long end) 308 { 309 unsigned long pfn; 310 void *entry; 311 312 /* 313 * The XArray doesn't hold references to pages since it relies on 314 * the mmu notifier to clear page pointers when they become stale. 315 * Therefore, it is OK to just clear the entry. 316 */ 317 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 318 end >> PAGE_SHIFT) 319 xa_erase(&dmirror->pt, pfn); 320 } 321 322 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 323 const struct mmu_notifier_range *range, 324 unsigned long cur_seq) 325 { 326 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 327 328 /* 329 * Ignore invalidation callbacks for device private pages since 330 * the invalidation is handled as part of the migration process. 331 */ 332 if (range->event == MMU_NOTIFY_MIGRATE && 333 range->owner == dmirror->mdevice) 334 return true; 335 336 if (mmu_notifier_range_blockable(range)) 337 mutex_lock(&dmirror->mutex); 338 else if (!mutex_trylock(&dmirror->mutex)) 339 return false; 340 341 mmu_interval_set_seq(mni, cur_seq); 342 dmirror_do_update(dmirror, range->start, range->end); 343 344 mutex_unlock(&dmirror->mutex); 345 return true; 346 } 347 348 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 349 .invalidate = dmirror_interval_invalidate, 350 }; 351 352 static int dmirror_range_fault(struct dmirror *dmirror, 353 struct hmm_range *range) 354 { 355 struct mm_struct *mm = dmirror->notifier.mm; 356 unsigned long timeout = 357 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 358 int ret; 359 360 while (true) { 361 if (time_after(jiffies, timeout)) { 362 ret = -EBUSY; 363 goto out; 364 } 365 366 range->notifier_seq = mmu_interval_read_begin(range->notifier); 367 mmap_read_lock(mm); 368 ret = hmm_range_fault(range); 369 mmap_read_unlock(mm); 370 if (ret) { 371 if (ret == -EBUSY) 372 continue; 373 goto out; 374 } 375 376 mutex_lock(&dmirror->mutex); 377 if (mmu_interval_read_retry(range->notifier, 378 range->notifier_seq)) { 379 mutex_unlock(&dmirror->mutex); 380 continue; 381 } 382 break; 383 } 384 385 ret = dmirror_do_fault(dmirror, range); 386 387 mutex_unlock(&dmirror->mutex); 388 out: 389 return ret; 390 } 391 392 static int dmirror_range_fault_unlocked(struct dmirror *dmirror, 393 struct hmm_range *range, 394 unsigned long timeout) 395 { 396 int ret; 397 398 while (true) { 399 ret = hmm_range_fault_unlocked_timeout(range, timeout); 400 if (ret) 401 goto out; 402 403 mutex_lock(&dmirror->mutex); 404 if (mmu_interval_read_retry(range->notifier, 405 range->notifier_seq)) { 406 mutex_unlock(&dmirror->mutex); 407 continue; 408 } 409 break; 410 } 411 412 ret = dmirror_do_fault(dmirror, range); 413 414 mutex_unlock(&dmirror->mutex); 415 out: 416 return ret; 417 } 418 419 static int dmirror_fault_unlocked(struct dmirror *dmirror, 420 unsigned long start, 421 unsigned long end, bool write, 422 unsigned long timeout) 423 { 424 struct mm_struct *mm = dmirror->notifier.mm; 425 unsigned long addr; 426 unsigned long pfns[32]; 427 struct hmm_range range = { 428 .notifier = &dmirror->notifier, 429 .hmm_pfns = pfns, 430 .pfn_flags_mask = 0, 431 .default_flags = 432 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 433 .dev_private_owner = dmirror->mdevice, 434 }; 435 int ret = 0; 436 437 if (!mmget_not_zero(mm)) 438 return -EFAULT; 439 440 for (addr = start; addr < end; addr = range.end) { 441 range.start = addr; 442 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 443 444 ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); 445 if (ret) 446 break; 447 } 448 449 mmput(mm); 450 return ret; 451 } 452 453 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 454 unsigned long end, bool write) 455 { 456 struct mm_struct *mm = dmirror->notifier.mm; 457 unsigned long addr; 458 unsigned long pfns[32]; 459 struct hmm_range range = { 460 .notifier = &dmirror->notifier, 461 .hmm_pfns = pfns, 462 .pfn_flags_mask = 0, 463 .default_flags = 464 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 465 .dev_private_owner = dmirror->mdevice, 466 }; 467 int ret = 0; 468 469 /* Since the mm is for the mirrored process, get a reference first. */ 470 if (!mmget_not_zero(mm)) 471 return -EFAULT; 472 473 for (addr = start; addr < end; addr = range.end) { 474 range.start = addr; 475 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 476 477 ret = dmirror_range_fault(dmirror, &range); 478 if (ret) 479 break; 480 } 481 482 mmput(mm); 483 return ret; 484 } 485 486 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 487 unsigned long end, struct dmirror_bounce *bounce) 488 { 489 unsigned long pfn; 490 void *ptr; 491 492 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 493 494 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 495 void *entry; 496 struct page *page; 497 498 entry = xa_load(&dmirror->pt, pfn); 499 page = xa_untag_pointer(entry); 500 if (!page) 501 return -ENOENT; 502 503 memcpy_from_page(ptr, page, 0, PAGE_SIZE); 504 505 ptr += PAGE_SIZE; 506 bounce->cpages++; 507 } 508 509 return 0; 510 } 511 512 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 513 { 514 struct dmirror_bounce bounce; 515 unsigned long start, end; 516 unsigned long size = cmd->npages << PAGE_SHIFT; 517 int ret; 518 519 start = cmd->addr; 520 end = start + size; 521 if (end < start) 522 return -EINVAL; 523 524 ret = dmirror_bounce_init(&bounce, start, size); 525 if (ret) 526 return ret; 527 528 while (1) { 529 mutex_lock(&dmirror->mutex); 530 ret = dmirror_do_read(dmirror, start, end, &bounce); 531 mutex_unlock(&dmirror->mutex); 532 if (ret != -ENOENT) 533 break; 534 535 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 536 ret = dmirror_fault(dmirror, start, end, false); 537 if (ret) 538 break; 539 cmd->faults++; 540 } 541 542 if (ret == 0) { 543 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 544 bounce.size)) 545 ret = -EFAULT; 546 } 547 cmd->cpages = bounce.cpages; 548 dmirror_bounce_fini(&bounce); 549 return ret; 550 } 551 552 static int dmirror_read_unlocked(struct dmirror *dmirror, 553 struct hmm_dmirror_cmd *cmd, 554 unsigned long timeout) 555 { 556 struct dmirror_bounce bounce; 557 unsigned long start, end; 558 unsigned long size = cmd->npages << PAGE_SHIFT; 559 int ret; 560 561 start = cmd->addr; 562 end = start + size; 563 if (end < start) 564 return -EINVAL; 565 566 ret = dmirror_bounce_init(&bounce, start, size); 567 if (ret) 568 return ret; 569 570 while (1) { 571 mutex_lock(&dmirror->mutex); 572 ret = dmirror_do_read(dmirror, start, end, &bounce); 573 mutex_unlock(&dmirror->mutex); 574 if (ret != -ENOENT) 575 break; 576 577 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 578 ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); 579 if (ret) 580 break; 581 cmd->faults++; 582 } 583 584 if (ret == 0) { 585 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 586 bounce.size)) 587 ret = -EFAULT; 588 } 589 cmd->cpages = bounce.cpages; 590 dmirror_bounce_fini(&bounce); 591 return ret; 592 } 593 594 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 595 unsigned long end, struct dmirror_bounce *bounce) 596 { 597 unsigned long pfn; 598 void *ptr; 599 600 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 601 602 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 603 void *entry; 604 struct page *page; 605 606 entry = xa_load(&dmirror->pt, pfn); 607 page = xa_untag_pointer(entry); 608 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 609 return -ENOENT; 610 611 memcpy_to_page(page, 0, ptr, PAGE_SIZE); 612 613 ptr += PAGE_SIZE; 614 bounce->cpages++; 615 } 616 617 return 0; 618 } 619 620 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 621 { 622 struct dmirror_bounce bounce; 623 unsigned long start, end; 624 unsigned long size = cmd->npages << PAGE_SHIFT; 625 int ret; 626 627 start = cmd->addr; 628 end = start + size; 629 if (end < start) 630 return -EINVAL; 631 632 ret = dmirror_bounce_init(&bounce, start, size); 633 if (ret) 634 return ret; 635 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 636 bounce.size)) { 637 ret = -EFAULT; 638 goto fini; 639 } 640 641 while (1) { 642 mutex_lock(&dmirror->mutex); 643 ret = dmirror_do_write(dmirror, start, end, &bounce); 644 mutex_unlock(&dmirror->mutex); 645 if (ret != -ENOENT) 646 break; 647 648 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 649 ret = dmirror_fault(dmirror, start, end, true); 650 if (ret) 651 break; 652 cmd->faults++; 653 } 654 655 fini: 656 cmd->cpages = bounce.cpages; 657 dmirror_bounce_fini(&bounce); 658 return ret; 659 } 660 661 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 662 struct page **ppage, bool is_large) 663 { 664 struct dmirror_chunk *devmem; 665 struct resource *res = NULL; 666 unsigned long pfn; 667 unsigned long pfn_first; 668 unsigned long pfn_last; 669 void *ptr; 670 int ret = -ENOMEM; 671 672 devmem = kzalloc_obj(*devmem); 673 if (!devmem) 674 return ret; 675 676 switch (mdevice->zone_device_type) { 677 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 678 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 679 "hmm_dmirror"); 680 if (IS_ERR_OR_NULL(res)) 681 goto err_devmem; 682 devmem->pagemap.range.start = res->start; 683 devmem->pagemap.range.end = res->end; 684 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 685 break; 686 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 687 devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? 688 spm_addr_dev0 : 689 spm_addr_dev1; 690 devmem->pagemap.range.end = devmem->pagemap.range.start + 691 DEVMEM_CHUNK_SIZE - 1; 692 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 693 break; 694 default: 695 ret = -EINVAL; 696 goto err_devmem; 697 } 698 699 devmem->pagemap.nr_range = 1; 700 devmem->pagemap.ops = &dmirror_devmem_ops; 701 devmem->pagemap.owner = mdevice; 702 703 mutex_lock(&mdevice->devmem_lock); 704 705 if (mdevice->devmem_count == mdevice->devmem_capacity) { 706 struct dmirror_chunk **new_chunks; 707 unsigned int new_capacity; 708 709 new_capacity = mdevice->devmem_capacity + 710 DEVMEM_CHUNKS_RESERVE; 711 new_chunks = krealloc(mdevice->devmem_chunks, 712 sizeof(new_chunks[0]) * new_capacity, 713 GFP_KERNEL); 714 if (!new_chunks) 715 goto err_release; 716 mdevice->devmem_capacity = new_capacity; 717 mdevice->devmem_chunks = new_chunks; 718 } 719 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 720 if (IS_ERR_OR_NULL(ptr)) { 721 if (ptr) 722 ret = PTR_ERR(ptr); 723 else 724 ret = -EFAULT; 725 goto err_release; 726 } 727 728 devmem->mdevice = mdevice; 729 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 730 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 731 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 732 733 mutex_unlock(&mdevice->devmem_lock); 734 735 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 736 DEVMEM_CHUNK_SIZE / (1024 * 1024), 737 mdevice->devmem_count, 738 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 739 pfn_first, pfn_last); 740 741 spin_lock(&mdevice->lock); 742 for (pfn = pfn_first; pfn < pfn_last; ) { 743 struct page *page = pfn_to_page(pfn); 744 745 if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR) 746 && (pfn + HPAGE_PMD_NR <= pfn_last)) { 747 page->zone_device_data = mdevice->free_folios; 748 mdevice->free_folios = page_folio(page); 749 pfn += HPAGE_PMD_NR; 750 continue; 751 } 752 753 page->zone_device_data = mdevice->free_pages; 754 mdevice->free_pages = page; 755 pfn++; 756 } 757 758 ret = 0; 759 if (ppage) { 760 if (is_large) { 761 if (!mdevice->free_folios) { 762 ret = -ENOMEM; 763 goto err_unlock; 764 } 765 *ppage = folio_page(mdevice->free_folios, 0); 766 mdevice->free_folios = (*ppage)->zone_device_data; 767 mdevice->calloc += HPAGE_PMD_NR; 768 } else if (mdevice->free_pages) { 769 *ppage = mdevice->free_pages; 770 mdevice->free_pages = (*ppage)->zone_device_data; 771 mdevice->calloc++; 772 } else { 773 ret = -ENOMEM; 774 goto err_unlock; 775 } 776 } 777 err_unlock: 778 spin_unlock(&mdevice->lock); 779 780 return ret; 781 782 err_release: 783 mutex_unlock(&mdevice->devmem_lock); 784 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 785 release_mem_region(devmem->pagemap.range.start, 786 range_len(&devmem->pagemap.range)); 787 err_devmem: 788 kfree(devmem); 789 790 return ret; 791 } 792 793 static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, 794 bool is_large) 795 { 796 struct page *dpage = NULL; 797 struct page *rpage = NULL; 798 unsigned int order = is_large ? HPAGE_PMD_ORDER : 0; 799 struct dmirror_device *mdevice = dmirror->mdevice; 800 801 /* 802 * For ZONE_DEVICE private type, this is a fake device so we allocate 803 * real system memory to store our device memory. 804 * For ZONE_DEVICE coherent type we use the actual dpage to store the 805 * data and ignore rpage. 806 */ 807 if (dmirror_is_private_zone(mdevice)) { 808 rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0); 809 if (!rpage) 810 return NULL; 811 } 812 spin_lock(&mdevice->lock); 813 814 if (is_large && mdevice->free_folios) { 815 dpage = folio_page(mdevice->free_folios, 0); 816 mdevice->free_folios = dpage->zone_device_data; 817 mdevice->calloc += 1 << order; 818 spin_unlock(&mdevice->lock); 819 } else if (!is_large && mdevice->free_pages) { 820 dpage = mdevice->free_pages; 821 mdevice->free_pages = dpage->zone_device_data; 822 mdevice->calloc++; 823 spin_unlock(&mdevice->lock); 824 } else { 825 spin_unlock(&mdevice->lock); 826 if (dmirror_allocate_chunk(mdevice, &dpage, is_large)) 827 goto error; 828 } 829 830 zone_device_folio_init(page_folio(dpage), 831 page_pgmap(folio_page(page_folio(dpage), 0)), 832 order); 833 dpage->zone_device_data = rpage; 834 return dpage; 835 836 error: 837 if (rpage) 838 __free_pages(rpage, order); 839 return NULL; 840 } 841 842 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 843 struct dmirror *dmirror) 844 { 845 const unsigned long *src = args->src; 846 unsigned long *dst = args->dst; 847 unsigned long addr; 848 849 for (addr = args->start; addr < args->end; ) { 850 struct page *spage; 851 struct page *dpage; 852 struct page *rpage; 853 bool is_large = *src & MIGRATE_PFN_COMPOUND; 854 int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0; 855 unsigned long nr = 1; 856 857 if (!(*src & MIGRATE_PFN_MIGRATE)) 858 goto next; 859 860 /* 861 * Note that spage might be NULL which is OK since it is an 862 * unallocated pte_none() or read-only zero page. 863 */ 864 spage = migrate_pfn_to_page(*src); 865 if (WARN(spage && is_zone_device_page(spage), 866 "page already in device spage pfn: 0x%lx\n", 867 page_to_pfn(spage))) 868 goto next; 869 870 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { 871 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; 872 dpage = NULL; 873 } else 874 dpage = dmirror_devmem_alloc_page(dmirror, is_large); 875 876 if (!dpage) { 877 struct folio *folio; 878 unsigned long i; 879 unsigned long spfn = *src >> MIGRATE_PFN_SHIFT; 880 struct page *src_page; 881 882 if (!is_large) 883 goto next; 884 885 if (!spage && is_large) { 886 nr = HPAGE_PMD_NR; 887 } else { 888 folio = page_folio(spage); 889 nr = folio_nr_pages(folio); 890 } 891 892 for (i = 0; i < nr && addr < args->end; i++) { 893 dpage = dmirror_devmem_alloc_page(dmirror, false); 894 rpage = BACKING_PAGE(dpage); 895 rpage->zone_device_data = dmirror; 896 897 *dst = migrate_pfn(page_to_pfn(dpage)) | write; 898 src_page = pfn_to_page(spfn + i); 899 900 if (spage) 901 copy_highpage(rpage, src_page); 902 else 903 clear_highpage(rpage); 904 src++; 905 dst++; 906 addr += PAGE_SIZE; 907 } 908 continue; 909 } 910 911 rpage = BACKING_PAGE(dpage); 912 913 /* 914 * Normally, a device would use the page->zone_device_data to 915 * point to the mirror but here we use it to hold the page for 916 * the simulated device memory and that page holds the pointer 917 * to the mirror. 918 */ 919 rpage->zone_device_data = dmirror; 920 921 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 922 page_to_pfn(spage), page_to_pfn(dpage)); 923 924 *dst = migrate_pfn(page_to_pfn(dpage)) | write; 925 926 if (is_large) { 927 int i; 928 struct folio *folio = page_folio(dpage); 929 *dst |= MIGRATE_PFN_COMPOUND; 930 931 if (folio_test_large(folio)) { 932 for (i = 0; i < folio_nr_pages(folio); i++) { 933 struct page *dst_page = 934 pfn_to_page(page_to_pfn(rpage) + i); 935 struct page *src_page = 936 pfn_to_page(page_to_pfn(spage) + i); 937 938 if (spage) 939 copy_highpage(dst_page, src_page); 940 else 941 clear_highpage(dst_page); 942 src++; 943 dst++; 944 addr += PAGE_SIZE; 945 } 946 continue; 947 } 948 } 949 950 if (spage) 951 copy_highpage(rpage, spage); 952 else 953 clear_highpage(rpage); 954 955 next: 956 src++; 957 dst++; 958 addr += PAGE_SIZE; 959 } 960 } 961 962 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 963 unsigned long end) 964 { 965 unsigned long pfn; 966 967 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 968 void *entry; 969 970 entry = xa_load(&dmirror->pt, pfn); 971 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 972 return -EPERM; 973 } 974 975 return 0; 976 } 977 978 static int dmirror_atomic_map(unsigned long addr, struct page *page, 979 struct dmirror *dmirror) 980 { 981 void *entry; 982 983 /* Map the migrated pages into the device's page tables. */ 984 mutex_lock(&dmirror->mutex); 985 986 entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC); 987 entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); 988 if (xa_is_err(entry)) { 989 mutex_unlock(&dmirror->mutex); 990 return xa_err(entry); 991 } 992 993 mutex_unlock(&dmirror->mutex); 994 return 0; 995 } 996 997 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 998 struct dmirror *dmirror) 999 { 1000 unsigned long start = args->start; 1001 unsigned long end = args->end; 1002 const unsigned long *src = args->src; 1003 const unsigned long *dst = args->dst; 1004 unsigned long pfn; 1005 const unsigned long start_pfn = start >> PAGE_SHIFT; 1006 const unsigned long end_pfn = end >> PAGE_SHIFT; 1007 1008 /* Map the migrated pages into the device's page tables. */ 1009 mutex_lock(&dmirror->mutex); 1010 1011 for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) { 1012 struct page *dpage; 1013 void *entry; 1014 int nr, i; 1015 struct page *rpage; 1016 1017 if (!(*src & MIGRATE_PFN_MIGRATE)) 1018 continue; 1019 1020 dpage = migrate_pfn_to_page(*dst); 1021 if (!dpage) 1022 continue; 1023 1024 if (*dst & MIGRATE_PFN_COMPOUND) 1025 nr = folio_nr_pages(page_folio(dpage)); 1026 else 1027 nr = 1; 1028 1029 WARN_ON_ONCE(end_pfn < start_pfn + nr); 1030 1031 rpage = BACKING_PAGE(dpage); 1032 VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr); 1033 1034 for (i = 0; i < nr; i++) { 1035 entry = folio_page(page_folio(rpage), i); 1036 if (*dst & MIGRATE_PFN_WRITE) 1037 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 1038 entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC); 1039 if (xa_is_err(entry)) { 1040 mutex_unlock(&dmirror->mutex); 1041 return xa_err(entry); 1042 } 1043 } 1044 } 1045 1046 mutex_unlock(&dmirror->mutex); 1047 return 0; 1048 } 1049 1050 static int dmirror_exclusive(struct dmirror *dmirror, 1051 struct hmm_dmirror_cmd *cmd) 1052 { 1053 unsigned long start, end, addr; 1054 unsigned long size = cmd->npages << PAGE_SHIFT; 1055 struct mm_struct *mm = dmirror->notifier.mm; 1056 struct dmirror_bounce bounce; 1057 int ret = 0; 1058 1059 start = cmd->addr; 1060 end = start + size; 1061 if (end < start) 1062 return -EINVAL; 1063 1064 /* Since the mm is for the mirrored process, get a reference first. */ 1065 if (!mmget_not_zero(mm)) 1066 return -EINVAL; 1067 1068 mmap_read_lock(mm); 1069 for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { 1070 struct folio *folio; 1071 struct page *page; 1072 1073 page = make_device_exclusive(mm, addr, NULL, &folio); 1074 if (IS_ERR(page)) { 1075 ret = PTR_ERR(page); 1076 break; 1077 } 1078 1079 ret = dmirror_atomic_map(addr, page, dmirror); 1080 folio_unlock(folio); 1081 folio_put(folio); 1082 } 1083 mmap_read_unlock(mm); 1084 mmput(mm); 1085 1086 if (ret) 1087 return ret; 1088 1089 /* Return the migrated data for verification. */ 1090 ret = dmirror_bounce_init(&bounce, start, size); 1091 if (ret) 1092 return ret; 1093 mutex_lock(&dmirror->mutex); 1094 ret = dmirror_do_read(dmirror, start, end, &bounce); 1095 mutex_unlock(&dmirror->mutex); 1096 if (ret == 0) { 1097 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1098 bounce.size)) 1099 ret = -EFAULT; 1100 } 1101 1102 cmd->cpages = bounce.cpages; 1103 dmirror_bounce_fini(&bounce); 1104 return ret; 1105 } 1106 1107 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 1108 struct dmirror *dmirror) 1109 { 1110 const unsigned long *src = args->src; 1111 unsigned long *dst = args->dst; 1112 unsigned long start = args->start; 1113 unsigned long end = args->end; 1114 unsigned long addr; 1115 unsigned int order = 0; 1116 int i; 1117 1118 for (addr = start; addr < end; ) { 1119 struct page *dpage, *spage; 1120 1121 spage = migrate_pfn_to_page(*src); 1122 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) { 1123 addr += PAGE_SIZE; 1124 goto next; 1125 } 1126 1127 if (WARN_ON(!is_device_private_page(spage) && 1128 !is_device_coherent_page(spage))) { 1129 addr += PAGE_SIZE; 1130 goto next; 1131 } 1132 1133 spage = BACKING_PAGE(spage); 1134 order = folio_order(page_folio(spage)); 1135 if (order) 1136 *dst = MIGRATE_PFN_COMPOUND; 1137 if (*src & MIGRATE_PFN_WRITE) 1138 *dst |= MIGRATE_PFN_WRITE; 1139 1140 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { 1141 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; 1142 *dst &= ~MIGRATE_PFN_COMPOUND; 1143 dpage = NULL; 1144 } else if (order) { 1145 dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 1146 order, args->vma, addr), 0); 1147 } else { 1148 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 1149 } 1150 1151 if (!dpage && !order) 1152 return VM_FAULT_OOM; 1153 1154 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 1155 page_to_pfn(spage), page_to_pfn(dpage)); 1156 1157 if (dpage) { 1158 lock_page(dpage); 1159 *dst |= migrate_pfn(page_to_pfn(dpage)); 1160 } 1161 1162 for (i = 0; i < (1 << order); i++) { 1163 struct page *src_page; 1164 struct page *dst_page; 1165 1166 /* Try with smaller pages if large allocation fails */ 1167 if (!dpage && order) { 1168 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 1169 if (!dpage) { 1170 /* Unlock and free pages already allocated. */ 1171 while (i > 0) { 1172 struct page *fpage; 1173 1174 fpage = migrate_pfn_to_page(dst[--i]); 1175 unlock_page(fpage); 1176 __free_page(fpage); 1177 } 1178 /* Clear remaining dst entries to avoid 1179 * migrate_vma_pages/finalize() using 1180 * uninitialized values. 1181 */ 1182 while (i < (1 << order)) { 1183 dst[i] = 0; 1184 i++; 1185 } 1186 return VM_FAULT_OOM; 1187 } 1188 lock_page(dpage); 1189 dst[i] = migrate_pfn(page_to_pfn(dpage)); 1190 dst_page = pfn_to_page(page_to_pfn(dpage)); 1191 dpage = NULL; /* For the next iteration */ 1192 } else { 1193 dst_page = pfn_to_page(page_to_pfn(dpage) + i); 1194 } 1195 1196 src_page = pfn_to_page(page_to_pfn(spage) + i); 1197 1198 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 1199 addr += PAGE_SIZE; 1200 copy_highpage(dst_page, src_page); 1201 } 1202 next: 1203 src += 1 << order; 1204 dst += 1 << order; 1205 } 1206 return 0; 1207 } 1208 1209 static unsigned long 1210 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 1211 { 1212 unsigned long cpages = 0; 1213 unsigned long i; 1214 1215 for (i = 0; i < migrate->npages; i++) { 1216 if (migrate->src[i] & MIGRATE_PFN_VALID && 1217 migrate->src[i] & MIGRATE_PFN_MIGRATE) 1218 cpages++; 1219 } 1220 return cpages; 1221 } 1222 1223 static int dmirror_migrate_to_system(struct dmirror *dmirror, 1224 struct hmm_dmirror_cmd *cmd) 1225 { 1226 unsigned long start, end, addr; 1227 unsigned long size = cmd->npages << PAGE_SHIFT; 1228 struct mm_struct *mm = dmirror->notifier.mm; 1229 struct vm_area_struct *vma; 1230 struct migrate_vma args = { 0 }; 1231 unsigned long next; 1232 int ret; 1233 unsigned long *src_pfns; 1234 unsigned long *dst_pfns; 1235 1236 start = cmd->addr; 1237 end = start + size; 1238 if (end < start) 1239 return -EINVAL; 1240 1241 /* Since the mm is for the mirrored process, get a reference first. */ 1242 if (!mmget_not_zero(mm)) 1243 return -EINVAL; 1244 1245 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 1246 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 1247 1248 cmd->cpages = 0; 1249 mmap_read_lock(mm); 1250 for (addr = start; addr < end; addr = next) { 1251 vma = vma_lookup(mm, addr); 1252 if (!vma || !(vma->vm_flags & VM_READ)) { 1253 ret = -EINVAL; 1254 goto out; 1255 } 1256 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); 1257 if (next > vma->vm_end) 1258 next = vma->vm_end; 1259 1260 args.vma = vma; 1261 args.src = src_pfns; 1262 args.dst = dst_pfns; 1263 args.start = addr; 1264 args.end = next; 1265 args.pgmap_owner = dmirror->mdevice; 1266 args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND; 1267 1268 ret = migrate_vma_setup(&args); 1269 if (ret) 1270 goto out; 1271 1272 pr_debug("Migrating from device mem to sys mem\n"); 1273 if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { 1274 migrate_vma_finalize(&args); 1275 ret = -ENOMEM; 1276 goto out; 1277 } 1278 1279 migrate_vma_pages(&args); 1280 cmd->cpages += dmirror_successful_migrated_pages(&args); 1281 migrate_vma_finalize(&args); 1282 } 1283 out: 1284 mmap_read_unlock(mm); 1285 mmput(mm); 1286 kvfree(src_pfns); 1287 kvfree(dst_pfns); 1288 1289 return ret; 1290 } 1291 1292 static int dmirror_migrate_to_device(struct dmirror *dmirror, 1293 struct hmm_dmirror_cmd *cmd) 1294 { 1295 unsigned long start, end, addr; 1296 unsigned long size = cmd->npages << PAGE_SHIFT; 1297 struct mm_struct *mm = dmirror->notifier.mm; 1298 struct vm_area_struct *vma; 1299 struct dmirror_bounce bounce; 1300 struct migrate_vma args = { 0 }; 1301 unsigned long next; 1302 int ret; 1303 unsigned long *src_pfns = NULL; 1304 unsigned long *dst_pfns = NULL; 1305 1306 start = cmd->addr; 1307 end = start + size; 1308 if (end < start) 1309 return -EINVAL; 1310 1311 /* Since the mm is for the mirrored process, get a reference first. */ 1312 if (!mmget_not_zero(mm)) 1313 return -EINVAL; 1314 1315 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), 1316 GFP_KERNEL | __GFP_NOFAIL); 1317 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), 1318 GFP_KERNEL | __GFP_NOFAIL); 1319 1320 ret = 0; 1321 mmap_read_lock(mm); 1322 for (addr = start; addr < end; addr = next) { 1323 vma = vma_lookup(mm, addr); 1324 if (!vma || !(vma->vm_flags & VM_READ)) { 1325 ret = -EINVAL; 1326 goto out; 1327 } 1328 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); 1329 if (next > vma->vm_end) 1330 next = vma->vm_end; 1331 1332 args.vma = vma; 1333 args.src = src_pfns; 1334 args.dst = dst_pfns; 1335 args.start = addr; 1336 args.end = next; 1337 args.pgmap_owner = dmirror->mdevice; 1338 args.flags = MIGRATE_VMA_SELECT_SYSTEM | 1339 MIGRATE_VMA_SELECT_COMPOUND; 1340 ret = migrate_vma_setup(&args); 1341 if (ret) 1342 goto out; 1343 1344 pr_debug("Migrating from sys mem to device mem\n"); 1345 dmirror_migrate_alloc_and_copy(&args, dmirror); 1346 migrate_vma_pages(&args); 1347 dmirror_migrate_finalize_and_map(&args, dmirror); 1348 migrate_vma_finalize(&args); 1349 } 1350 mmap_read_unlock(mm); 1351 mmput(mm); 1352 1353 /* 1354 * Return the migrated data for verification. 1355 * Only for pages in device zone 1356 */ 1357 ret = dmirror_bounce_init(&bounce, start, size); 1358 if (ret) 1359 goto free_mem; 1360 mutex_lock(&dmirror->mutex); 1361 ret = dmirror_do_read(dmirror, start, end, &bounce); 1362 mutex_unlock(&dmirror->mutex); 1363 if (ret == 0) { 1364 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1365 bounce.size)) 1366 ret = -EFAULT; 1367 } 1368 cmd->cpages = bounce.cpages; 1369 dmirror_bounce_fini(&bounce); 1370 goto free_mem; 1371 1372 out: 1373 mmap_read_unlock(mm); 1374 mmput(mm); 1375 free_mem: 1376 kvfree(src_pfns); 1377 kvfree(dst_pfns); 1378 return ret; 1379 } 1380 1381 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1382 unsigned char *perm, unsigned long entry) 1383 { 1384 struct page *page; 1385 1386 if (entry & HMM_PFN_ERROR) { 1387 *perm = HMM_DMIRROR_PROT_ERROR; 1388 return; 1389 } 1390 if (!(entry & HMM_PFN_VALID)) { 1391 *perm = HMM_DMIRROR_PROT_NONE; 1392 return; 1393 } 1394 1395 page = hmm_pfn_to_page(entry); 1396 if (is_device_private_page(page)) { 1397 /* Is the page migrated to this device or some other? */ 1398 if (dmirror->mdevice == dmirror_page_to_device(page)) 1399 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1400 else 1401 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1402 } else if (is_device_coherent_page(page)) { 1403 /* Is the page migrated to this device or some other? */ 1404 if (dmirror->mdevice == dmirror_page_to_device(page)) 1405 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1406 else 1407 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1408 } else if (is_zero_pfn(page_to_pfn(page))) 1409 *perm = HMM_DMIRROR_PROT_ZERO; 1410 else 1411 *perm = HMM_DMIRROR_PROT_NONE; 1412 if (entry & HMM_PFN_WRITE) 1413 *perm |= HMM_DMIRROR_PROT_WRITE; 1414 else 1415 *perm |= HMM_DMIRROR_PROT_READ; 1416 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1417 *perm |= HMM_DMIRROR_PROT_PMD; 1418 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1419 *perm |= HMM_DMIRROR_PROT_PUD; 1420 } 1421 1422 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1423 const struct mmu_notifier_range *range, 1424 unsigned long cur_seq) 1425 { 1426 struct dmirror_interval *dmi = 1427 container_of(mni, struct dmirror_interval, notifier); 1428 struct dmirror *dmirror = dmi->dmirror; 1429 1430 if (mmu_notifier_range_blockable(range)) 1431 mutex_lock(&dmirror->mutex); 1432 else if (!mutex_trylock(&dmirror->mutex)) 1433 return false; 1434 1435 /* 1436 * Snapshots only need to set the sequence number since any 1437 * invalidation in the interval invalidates the whole snapshot. 1438 */ 1439 mmu_interval_set_seq(mni, cur_seq); 1440 1441 mutex_unlock(&dmirror->mutex); 1442 return true; 1443 } 1444 1445 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1446 .invalidate = dmirror_snapshot_invalidate, 1447 }; 1448 1449 static int dmirror_range_snapshot(struct dmirror *dmirror, 1450 struct hmm_range *range, 1451 unsigned char *perm) 1452 { 1453 struct mm_struct *mm = dmirror->notifier.mm; 1454 struct dmirror_interval notifier; 1455 unsigned long timeout = 1456 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1457 unsigned long i; 1458 unsigned long n; 1459 int ret = 0; 1460 1461 notifier.dmirror = dmirror; 1462 range->notifier = ¬ifier.notifier; 1463 1464 ret = mmu_interval_notifier_insert(range->notifier, mm, 1465 range->start, range->end - range->start, 1466 &dmirror_mrn_ops); 1467 if (ret) 1468 return ret; 1469 1470 while (true) { 1471 if (time_after(jiffies, timeout)) { 1472 ret = -EBUSY; 1473 goto out; 1474 } 1475 1476 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1477 1478 mmap_read_lock(mm); 1479 ret = hmm_range_fault(range); 1480 mmap_read_unlock(mm); 1481 if (ret) { 1482 if (ret == -EBUSY) 1483 continue; 1484 goto out; 1485 } 1486 1487 mutex_lock(&dmirror->mutex); 1488 if (mmu_interval_read_retry(range->notifier, 1489 range->notifier_seq)) { 1490 mutex_unlock(&dmirror->mutex); 1491 continue; 1492 } 1493 break; 1494 } 1495 1496 n = (range->end - range->start) >> PAGE_SHIFT; 1497 for (i = 0; i < n; i++) 1498 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1499 1500 mutex_unlock(&dmirror->mutex); 1501 out: 1502 mmu_interval_notifier_remove(range->notifier); 1503 return ret; 1504 } 1505 1506 static int dmirror_snapshot(struct dmirror *dmirror, 1507 struct hmm_dmirror_cmd *cmd) 1508 { 1509 struct mm_struct *mm = dmirror->notifier.mm; 1510 unsigned long start, end; 1511 unsigned long size = cmd->npages << PAGE_SHIFT; 1512 unsigned long addr; 1513 unsigned long next; 1514 unsigned long pfns[32]; 1515 unsigned char perm[32]; 1516 char __user *uptr; 1517 struct hmm_range range = { 1518 .hmm_pfns = pfns, 1519 .dev_private_owner = dmirror->mdevice, 1520 }; 1521 int ret = 0; 1522 1523 start = cmd->addr; 1524 end = start + size; 1525 if (end < start) 1526 return -EINVAL; 1527 1528 /* Since the mm is for the mirrored process, get a reference first. */ 1529 if (!mmget_not_zero(mm)) 1530 return -EINVAL; 1531 1532 /* 1533 * Register a temporary notifier to detect invalidations even if it 1534 * overlaps with other mmu_interval_notifiers. 1535 */ 1536 uptr = u64_to_user_ptr(cmd->ptr); 1537 for (addr = start; addr < end; addr = next) { 1538 unsigned long n; 1539 1540 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1541 range.start = addr; 1542 range.end = next; 1543 1544 ret = dmirror_range_snapshot(dmirror, &range, perm); 1545 if (ret) 1546 break; 1547 1548 n = (range.end - range.start) >> PAGE_SHIFT; 1549 if (copy_to_user(uptr, perm, n)) { 1550 ret = -EFAULT; 1551 break; 1552 } 1553 1554 cmd->cpages += n; 1555 uptr += n; 1556 } 1557 mmput(mm); 1558 1559 return ret; 1560 } 1561 1562 /* Removes free pages from the free list so they can't be re-allocated */ 1563 static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) 1564 { 1565 struct dmirror_device *mdevice = devmem->mdevice; 1566 struct page *page; 1567 struct folio *folio; 1568 1569 1570 for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio)) 1571 if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem) 1572 mdevice->free_folios = folio_zone_device_data(folio); 1573 for (page = mdevice->free_pages; page; page = page->zone_device_data) 1574 if (dmirror_page_to_chunk(page) == devmem) 1575 mdevice->free_pages = page->zone_device_data; 1576 } 1577 1578 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) 1579 { 1580 unsigned int i; 1581 1582 mutex_lock(&mdevice->devmem_lock); 1583 if (mdevice->devmem_chunks) { 1584 for (i = 0; i < mdevice->devmem_count; i++) { 1585 struct dmirror_chunk *devmem = 1586 mdevice->devmem_chunks[i]; 1587 1588 spin_lock(&mdevice->lock); 1589 devmem->remove = true; 1590 dmirror_remove_free_pages(devmem); 1591 spin_unlock(&mdevice->lock); 1592 1593 dmirror_device_evict_chunk(devmem); 1594 memunmap_pages(&devmem->pagemap); 1595 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1596 release_mem_region(devmem->pagemap.range.start, 1597 range_len(&devmem->pagemap.range)); 1598 kfree(devmem); 1599 } 1600 mdevice->devmem_count = 0; 1601 mdevice->devmem_capacity = 0; 1602 mdevice->free_pages = NULL; 1603 mdevice->free_folios = NULL; 1604 kfree(mdevice->devmem_chunks); 1605 mdevice->devmem_chunks = NULL; 1606 } 1607 mutex_unlock(&mdevice->devmem_lock); 1608 } 1609 1610 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1611 unsigned int command, 1612 unsigned long arg) 1613 { 1614 void __user *uarg = (void __user *)arg; 1615 struct hmm_dmirror_cmd cmd; 1616 struct dmirror *dmirror; 1617 int ret; 1618 1619 dmirror = filp->private_data; 1620 if (!dmirror) 1621 return -EINVAL; 1622 1623 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1624 return -EFAULT; 1625 1626 if (cmd.addr & ~PAGE_MASK) 1627 return -EINVAL; 1628 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1629 return -EINVAL; 1630 1631 cmd.cpages = 0; 1632 cmd.faults = 0; 1633 1634 switch (command) { 1635 case HMM_DMIRROR_READ: 1636 ret = dmirror_read(dmirror, &cmd); 1637 break; 1638 1639 case HMM_DMIRROR_WRITE: 1640 ret = dmirror_write(dmirror, &cmd); 1641 break; 1642 1643 case HMM_DMIRROR_MIGRATE_TO_DEV: 1644 ret = dmirror_migrate_to_device(dmirror, &cmd); 1645 break; 1646 1647 case HMM_DMIRROR_MIGRATE_TO_SYS: 1648 ret = dmirror_migrate_to_system(dmirror, &cmd); 1649 break; 1650 1651 case HMM_DMIRROR_EXCLUSIVE: 1652 ret = dmirror_exclusive(dmirror, &cmd); 1653 break; 1654 1655 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1656 ret = dmirror_check_atomic(dmirror, cmd.addr, 1657 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1658 break; 1659 1660 case HMM_DMIRROR_SNAPSHOT: 1661 ret = dmirror_snapshot(dmirror, &cmd); 1662 break; 1663 1664 case HMM_DMIRROR_RELEASE: 1665 dmirror_device_remove_chunks(dmirror->mdevice); 1666 ret = 0; 1667 break; 1668 case HMM_DMIRROR_FLAGS: 1669 dmirror->flags = cmd.npages; 1670 ret = 0; 1671 break; 1672 case HMM_DMIRROR_READ_UNLOCKED: 1673 ret = dmirror_read_unlocked(dmirror, &cmd, 0); 1674 break; 1675 default: 1676 return -EINVAL; 1677 } 1678 if (ret) 1679 return ret; 1680 1681 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1682 return -EFAULT; 1683 1684 return 0; 1685 } 1686 1687 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1688 { 1689 unsigned long addr; 1690 1691 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1692 struct page *page; 1693 int ret; 1694 1695 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1696 if (!page) 1697 return -ENOMEM; 1698 1699 ret = vm_insert_page(vma, addr, page); 1700 if (ret) { 1701 __free_page(page); 1702 return ret; 1703 } 1704 put_page(page); 1705 } 1706 1707 return 0; 1708 } 1709 1710 static const struct file_operations dmirror_fops = { 1711 .open = dmirror_fops_open, 1712 .release = dmirror_fops_release, 1713 .mmap = dmirror_fops_mmap, 1714 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1715 .llseek = default_llseek, 1716 .owner = THIS_MODULE, 1717 }; 1718 1719 static void dmirror_devmem_free(struct folio *folio) 1720 { 1721 struct page *page = &folio->page; 1722 struct page *rpage = BACKING_PAGE(page); 1723 struct dmirror_device *mdevice; 1724 struct folio *rfolio = page_folio(rpage); 1725 unsigned int order = folio_order(rfolio); 1726 1727 if (rpage != page) { 1728 if (order) 1729 __free_pages(rpage, order); 1730 else 1731 __free_page(rpage); 1732 rpage = NULL; 1733 } 1734 1735 mdevice = dmirror_page_to_device(page); 1736 spin_lock(&mdevice->lock); 1737 1738 /* Return page to our allocator if not freeing the chunk */ 1739 if (!dmirror_page_to_chunk(page)->remove) { 1740 mdevice->cfree += 1 << order; 1741 if (order) { 1742 page->zone_device_data = mdevice->free_folios; 1743 mdevice->free_folios = page_folio(page); 1744 } else { 1745 page->zone_device_data = mdevice->free_pages; 1746 mdevice->free_pages = page; 1747 } 1748 } 1749 spin_unlock(&mdevice->lock); 1750 } 1751 1752 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1753 { 1754 struct migrate_vma args = { 0 }; 1755 struct page *rpage; 1756 struct dmirror *dmirror; 1757 vm_fault_t ret = 0; 1758 unsigned int order, nr; 1759 1760 /* 1761 * Normally, a device would use the page->zone_device_data to point to 1762 * the mirror but here we use it to hold the page for the simulated 1763 * device memory and that page holds the pointer to the mirror. 1764 */ 1765 rpage = folio_zone_device_data(page_folio(vmf->page)); 1766 dmirror = rpage->zone_device_data; 1767 1768 /* FIXME demonstrate how we can adjust migrate range */ 1769 order = folio_order(page_folio(vmf->page)); 1770 nr = 1 << order; 1771 1772 /* 1773 * When folios are partially mapped, we can't rely on the folio 1774 * order of vmf->page as the folio might not be fully split yet 1775 */ 1776 if (vmf->pte) { 1777 order = 0; 1778 nr = 1; 1779 } 1780 1781 /* 1782 * Consider a per-cpu cache of src and dst pfns, but with 1783 * large number of cpus that might not scale well. 1784 */ 1785 args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order)); 1786 args.vma = vmf->vma; 1787 args.end = args.start + (PAGE_SIZE << order); 1788 1789 nr = (args.end - args.start) >> PAGE_SHIFT; 1790 args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); 1791 args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); 1792 args.pgmap_owner = dmirror->mdevice; 1793 args.flags = dmirror_select_device(dmirror); 1794 args.fault_page = vmf->page; 1795 1796 if (!args.src || !args.dst) { 1797 ret = VM_FAULT_OOM; 1798 goto err; 1799 } 1800 1801 if (order) 1802 args.flags |= MIGRATE_VMA_SELECT_COMPOUND; 1803 1804 /* 1805 * In practice migrate_vma_setup() should never fail unless the 1806 * test is wrong as it just tests some static VMA properties. 1807 */ 1808 if (migrate_vma_setup(&args)) { 1809 ret = VM_FAULT_SIGBUS; 1810 goto err; 1811 } 1812 1813 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1814 if (ret) { 1815 migrate_vma_finalize(&args); 1816 goto err; 1817 } 1818 migrate_vma_pages(&args); 1819 /* 1820 * No device finalize step is needed since 1821 * dmirror_devmem_fault_alloc_and_copy() will have already 1822 * invalidated the device page table. 1823 */ 1824 migrate_vma_finalize(&args); 1825 err: 1826 kfree(args.src); 1827 kfree(args.dst); 1828 return ret; 1829 } 1830 1831 static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail) 1832 { 1833 struct page *rpage = BACKING_PAGE(folio_page(head, 0)); 1834 struct page *rpage_tail; 1835 struct folio *rfolio; 1836 unsigned long offset = 0; 1837 1838 if (!rpage) { 1839 tail->page.zone_device_data = NULL; 1840 return; 1841 } 1842 1843 rfolio = page_folio(rpage); 1844 1845 if (tail == NULL) { 1846 folio_reset_order(rfolio); 1847 rfolio->mapping = NULL; 1848 folio_set_count(rfolio, 1); 1849 return; 1850 } 1851 1852 offset = folio_pfn(tail) - folio_pfn(head); 1853 1854 rpage_tail = folio_page(rfolio, offset); 1855 tail->page.zone_device_data = rpage_tail; 1856 rpage_tail->zone_device_data = rpage->zone_device_data; 1857 clear_compound_head(rpage_tail); 1858 rpage_tail->mapping = NULL; 1859 1860 folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping; 1861 tail->pgmap = head->pgmap; 1862 folio_set_count(page_folio(rpage_tail), 1); 1863 } 1864 1865 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1866 .folio_free = dmirror_devmem_free, 1867 .migrate_to_ram = dmirror_devmem_fault, 1868 .folio_split = dmirror_devmem_folio_split, 1869 }; 1870 1871 static void dmirror_device_release(struct device *dev) 1872 { 1873 struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device); 1874 1875 dmirror_device_remove_chunks(mdevice); 1876 } 1877 1878 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1879 { 1880 dev_t dev; 1881 int ret; 1882 1883 dev = MKDEV(MAJOR(dmirror_dev), id); 1884 mutex_init(&mdevice->devmem_lock); 1885 spin_lock_init(&mdevice->lock); 1886 1887 cdev_init(&mdevice->cdevice, &dmirror_fops); 1888 mdevice->cdevice.owner = THIS_MODULE; 1889 mdevice->device.release = dmirror_device_release; 1890 1891 device_initialize(&mdevice->device); 1892 mdevice->device.devt = dev; 1893 1894 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); 1895 if (ret) 1896 goto put_device; 1897 1898 /* Build a list of free ZONE_DEVICE struct pages */ 1899 ret = dmirror_allocate_chunk(mdevice, NULL, false); 1900 if (ret) 1901 goto put_device; 1902 1903 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); 1904 if (ret) 1905 goto put_device; 1906 1907 return 0; 1908 1909 put_device: 1910 put_device(&mdevice->device); 1911 return ret; 1912 } 1913 1914 static void dmirror_device_remove(struct dmirror_device *mdevice) 1915 { 1916 cdev_device_del(&mdevice->cdevice, &mdevice->device); 1917 put_device(&mdevice->device); 1918 } 1919 1920 static int __init hmm_dmirror_init(void) 1921 { 1922 int ret; 1923 int id = 0; 1924 int ndevices = 0; 1925 1926 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1927 "HMM_DMIRROR"); 1928 if (ret) 1929 goto err_unreg; 1930 1931 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1932 dmirror_devices[ndevices++].zone_device_type = 1933 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1934 dmirror_devices[ndevices++].zone_device_type = 1935 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1936 if (spm_addr_dev0 && spm_addr_dev1) { 1937 dmirror_devices[ndevices++].zone_device_type = 1938 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1939 dmirror_devices[ndevices++].zone_device_type = 1940 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1941 } 1942 for (id = 0; id < ndevices; id++) { 1943 ret = dmirror_device_init(dmirror_devices + id, id); 1944 if (ret) 1945 goto err_chrdev; 1946 } 1947 1948 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1949 return 0; 1950 1951 err_chrdev: 1952 while (--id >= 0) 1953 dmirror_device_remove(dmirror_devices + id); 1954 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1955 err_unreg: 1956 return ret; 1957 } 1958 1959 static void __exit hmm_dmirror_exit(void) 1960 { 1961 int id; 1962 1963 for (id = 0; id < DMIRROR_NDEVICES; id++) 1964 if (dmirror_devices[id].zone_device_type) 1965 dmirror_device_remove(dmirror_devices + id); 1966 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1967 } 1968 1969 module_init(hmm_dmirror_init); 1970 module_exit(hmm_dmirror_exit); 1971 MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module"); 1972 MODULE_LICENSE("GPL"); 1973