1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 __u64 flags; 96 }; 97 98 /* 99 * ZONE_DEVICE pages for migration and simulating device memory. 100 */ 101 struct dmirror_chunk { 102 struct dev_pagemap pagemap; 103 struct dmirror_device *mdevice; 104 bool remove; 105 }; 106 107 /* 108 * Per device data. 109 */ 110 struct dmirror_device { 111 struct cdev cdevice; 112 unsigned int zone_device_type; 113 struct device device; 114 115 unsigned int devmem_capacity; 116 unsigned int devmem_count; 117 struct dmirror_chunk **devmem_chunks; 118 struct mutex devmem_lock; /* protects the above */ 119 120 unsigned long calloc; 121 unsigned long cfree; 122 struct page *free_pages; 123 struct folio *free_folios; 124 spinlock_t lock; /* protects the above */ 125 }; 126 127 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 128 129 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 130 unsigned long addr, 131 unsigned long size) 132 { 133 bounce->addr = addr; 134 bounce->size = size; 135 bounce->cpages = 0; 136 bounce->ptr = vmalloc(size); 137 if (!bounce->ptr) 138 return -ENOMEM; 139 return 0; 140 } 141 142 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 143 { 144 return (mdevice->zone_device_type == 145 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE); 146 } 147 148 static enum migrate_vma_direction 149 dmirror_select_device(struct dmirror *dmirror) 150 { 151 return (dmirror->mdevice->zone_device_type == 152 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 153 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 154 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 155 } 156 157 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 158 { 159 vfree(bounce->ptr); 160 } 161 162 static int dmirror_fops_open(struct inode *inode, struct file *filp) 163 { 164 struct cdev *cdev = inode->i_cdev; 165 struct dmirror *dmirror; 166 int ret; 167 168 /* Mirror this process address space */ 169 dmirror = kzalloc_obj(*dmirror); 170 if (dmirror == NULL) 171 return -ENOMEM; 172 173 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 174 mutex_init(&dmirror->mutex); 175 xa_init(&dmirror->pt); 176 177 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 178 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 179 if (ret) { 180 kfree(dmirror); 181 return ret; 182 } 183 184 filp->private_data = dmirror; 185 return 0; 186 } 187 188 static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) 189 { 190 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; 191 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; 192 unsigned long npages = end_pfn - start_pfn + 1; 193 unsigned long i; 194 unsigned long *src_pfns; 195 unsigned long *dst_pfns; 196 unsigned int order = 0; 197 198 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 199 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 200 201 migrate_device_range(src_pfns, start_pfn, npages); 202 for (i = 0; i < npages; i++) { 203 struct page *dpage, *spage; 204 205 spage = migrate_pfn_to_page(src_pfns[i]); 206 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 207 continue; 208 209 if (WARN_ON(!is_device_private_page(spage) && 210 !is_device_coherent_page(spage))) 211 continue; 212 213 order = folio_order(page_folio(spage)); 214 spage = BACKING_PAGE(spage); 215 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { 216 dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, 217 order), 0); 218 } else { 219 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); 220 order = 0; 221 } 222 223 /* TODO Support splitting here */ 224 lock_page(dpage); 225 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 226 if (src_pfns[i] & MIGRATE_PFN_WRITE) 227 dst_pfns[i] |= MIGRATE_PFN_WRITE; 228 if (order) 229 dst_pfns[i] |= MIGRATE_PFN_COMPOUND; 230 folio_copy(page_folio(dpage), page_folio(spage)); 231 } 232 migrate_device_pages(src_pfns, dst_pfns, npages); 233 migrate_device_finalize(src_pfns, dst_pfns, npages); 234 kvfree(src_pfns); 235 kvfree(dst_pfns); 236 } 237 238 static int dmirror_fops_release(struct inode *inode, struct file *filp) 239 { 240 struct dmirror *dmirror = filp->private_data; 241 struct dmirror_device *mdevice = dmirror->mdevice; 242 int i; 243 244 mmu_interval_notifier_remove(&dmirror->notifier); 245 246 if (mdevice->devmem_chunks) { 247 for (i = 0; i < mdevice->devmem_count; i++) { 248 struct dmirror_chunk *devmem = 249 mdevice->devmem_chunks[i]; 250 251 dmirror_device_evict_chunk(devmem); 252 } 253 } 254 255 xa_destroy(&dmirror->pt); 256 kfree(dmirror); 257 return 0; 258 } 259 260 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) 261 { 262 return container_of(page_pgmap(page), struct dmirror_chunk, 263 pagemap); 264 } 265 266 static struct dmirror_device *dmirror_page_to_device(struct page *page) 267 268 { 269 return dmirror_page_to_chunk(page)->mdevice; 270 } 271 272 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 273 { 274 unsigned long *pfns = range->hmm_pfns; 275 unsigned long pfn; 276 277 for (pfn = (range->start >> PAGE_SHIFT); 278 pfn < (range->end >> PAGE_SHIFT); 279 pfn++, pfns++) { 280 struct page *page; 281 void *entry; 282 283 /* 284 * Since we asked for hmm_range_fault() to populate pages, 285 * it shouldn't return an error entry on success. 286 */ 287 WARN_ON(*pfns & HMM_PFN_ERROR); 288 WARN_ON(!(*pfns & HMM_PFN_VALID)); 289 290 page = hmm_pfn_to_page(*pfns); 291 WARN_ON(!page); 292 293 entry = page; 294 if (*pfns & HMM_PFN_WRITE) 295 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 296 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 297 return -EFAULT; 298 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 299 if (xa_is_err(entry)) 300 return xa_err(entry); 301 } 302 303 return 0; 304 } 305 306 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 307 unsigned long end) 308 { 309 unsigned long pfn; 310 void *entry; 311 312 /* 313 * The XArray doesn't hold references to pages since it relies on 314 * the mmu notifier to clear page pointers when they become stale. 315 * Therefore, it is OK to just clear the entry. 316 */ 317 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 318 end >> PAGE_SHIFT) 319 xa_erase(&dmirror->pt, pfn); 320 } 321 322 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 323 const struct mmu_notifier_range *range, 324 unsigned long cur_seq) 325 { 326 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 327 328 /* 329 * Ignore invalidation callbacks for device private pages since 330 * the invalidation is handled as part of the migration process. 331 */ 332 if (range->event == MMU_NOTIFY_MIGRATE && 333 range->owner == dmirror->mdevice) 334 return true; 335 336 if (mmu_notifier_range_blockable(range)) 337 mutex_lock(&dmirror->mutex); 338 else if (!mutex_trylock(&dmirror->mutex)) 339 return false; 340 341 mmu_interval_set_seq(mni, cur_seq); 342 dmirror_do_update(dmirror, range->start, range->end); 343 344 mutex_unlock(&dmirror->mutex); 345 return true; 346 } 347 348 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 349 .invalidate = dmirror_interval_invalidate, 350 }; 351 352 static int dmirror_range_fault(struct dmirror *dmirror, 353 struct hmm_range *range) 354 { 355 struct mm_struct *mm = dmirror->notifier.mm; 356 unsigned long timeout = 357 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 358 int ret; 359 360 while (true) { 361 if (time_after(jiffies, timeout)) { 362 ret = -EBUSY; 363 goto out; 364 } 365 366 range->notifier_seq = mmu_interval_read_begin(range->notifier); 367 mmap_read_lock(mm); 368 ret = hmm_range_fault(range); 369 mmap_read_unlock(mm); 370 if (ret) { 371 if (ret == -EBUSY) 372 continue; 373 goto out; 374 } 375 376 mutex_lock(&dmirror->mutex); 377 if (mmu_interval_read_retry(range->notifier, 378 range->notifier_seq)) { 379 mutex_unlock(&dmirror->mutex); 380 continue; 381 } 382 break; 383 } 384 385 ret = dmirror_do_fault(dmirror, range); 386 387 mutex_unlock(&dmirror->mutex); 388 out: 389 return ret; 390 } 391 392 static int dmirror_range_fault_unlocked(struct dmirror *dmirror, 393 struct hmm_range *range, 394 unsigned long timeout) 395 { 396 int ret; 397 398 while (true) { 399 ret = hmm_range_fault_unlocked_timeout(range, timeout); 400 if (ret) 401 goto out; 402 403 mutex_lock(&dmirror->mutex); 404 if (mmu_interval_read_retry(range->notifier, 405 range->notifier_seq)) { 406 mutex_unlock(&dmirror->mutex); 407 continue; 408 } 409 break; 410 } 411 412 ret = dmirror_do_fault(dmirror, range); 413 414 mutex_unlock(&dmirror->mutex); 415 out: 416 return ret; 417 } 418 419 static int dmirror_fault_unlocked(struct dmirror *dmirror, 420 unsigned long start, 421 unsigned long end, bool write, 422 unsigned long timeout) 423 { 424 struct mm_struct *mm = dmirror->notifier.mm; 425 unsigned long addr; 426 unsigned long pfns[32]; 427 struct hmm_range range = { 428 .notifier = &dmirror->notifier, 429 .hmm_pfns = pfns, 430 .pfn_flags_mask = 0, 431 .default_flags = 432 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 433 .dev_private_owner = dmirror->mdevice, 434 }; 435 int ret = 0; 436 437 if (!mmget_not_zero(mm)) 438 return -EFAULT; 439 440 for (addr = start; addr < end; addr = range.end) { 441 range.start = addr; 442 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 443 444 ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); 445 if (ret) 446 break; 447 } 448 449 mmput(mm); 450 return ret; 451 } 452 453 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 454 unsigned long end, bool write) 455 { 456 struct mm_struct *mm = dmirror->notifier.mm; 457 unsigned long addr; 458 unsigned long pfns[32]; 459 struct hmm_range range = { 460 .notifier = &dmirror->notifier, 461 .hmm_pfns = pfns, 462 .pfn_flags_mask = 0, 463 .default_flags = 464 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 465 .dev_private_owner = dmirror->mdevice, 466 }; 467 int ret = 0; 468 469 /* Since the mm is for the mirrored process, get a reference first. */ 470 if (!mmget_not_zero(mm)) 471 return -EFAULT; 472 473 for (addr = start; addr < end; addr = range.end) { 474 range.start = addr; 475 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 476 477 ret = dmirror_range_fault(dmirror, &range); 478 if (ret) 479 break; 480 } 481 482 mmput(mm); 483 return ret; 484 } 485 486 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 487 unsigned long end, struct dmirror_bounce *bounce) 488 { 489 unsigned long pfn; 490 void *ptr; 491 492 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 493 494 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 495 void *entry; 496 struct page *page; 497 498 entry = xa_load(&dmirror->pt, pfn); 499 page = xa_untag_pointer(entry); 500 if (!page) 501 return -ENOENT; 502 503 memcpy_from_page(ptr, page, 0, PAGE_SIZE); 504 505 ptr += PAGE_SIZE; 506 bounce->cpages++; 507 } 508 509 return 0; 510 } 511 512 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 513 { 514 struct dmirror_bounce bounce; 515 unsigned long start, end; 516 unsigned long size = cmd->npages << PAGE_SHIFT; 517 int ret; 518 519 start = cmd->addr; 520 end = start + size; 521 if (end < start) 522 return -EINVAL; 523 524 ret = dmirror_bounce_init(&bounce, start, size); 525 if (ret) 526 return ret; 527 528 while (1) { 529 mutex_lock(&dmirror->mutex); 530 ret = dmirror_do_read(dmirror, start, end, &bounce); 531 mutex_unlock(&dmirror->mutex); 532 if (ret != -ENOENT) 533 break; 534 535 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 536 ret = dmirror_fault(dmirror, start, end, false); 537 if (ret) 538 break; 539 cmd->faults++; 540 } 541 542 if (ret == 0) { 543 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 544 bounce.size)) 545 ret = -EFAULT; 546 } 547 cmd->cpages = bounce.cpages; 548 dmirror_bounce_fini(&bounce); 549 return ret; 550 } 551 552 static int dmirror_read_unlocked(struct dmirror *dmirror, 553 struct hmm_dmirror_cmd *cmd, 554 unsigned long timeout) 555 { 556 struct dmirror_bounce bounce; 557 unsigned long start, end; 558 unsigned long size = cmd->npages << PAGE_SHIFT; 559 int ret; 560 561 start = cmd->addr; 562 end = start + size; 563 if (end < start) 564 return -EINVAL; 565 566 ret = dmirror_bounce_init(&bounce, start, size); 567 if (ret) 568 return ret; 569 570 while (1) { 571 mutex_lock(&dmirror->mutex); 572 ret = dmirror_do_read(dmirror, start, end, &bounce); 573 mutex_unlock(&dmirror->mutex); 574 if (ret != -ENOENT) 575 break; 576 577 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 578 ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); 579 if (ret) 580 break; 581 cmd->faults++; 582 } 583 584 if (ret == 0) { 585 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 586 bounce.size)) 587 ret = -EFAULT; 588 } 589 cmd->cpages = bounce.cpages; 590 dmirror_bounce_fini(&bounce); 591 return ret; 592 } 593 594 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 595 unsigned long end, struct dmirror_bounce *bounce) 596 { 597 unsigned long pfn; 598 void *ptr; 599 600 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 601 602 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 603 void *entry; 604 struct page *page; 605 606 entry = xa_load(&dmirror->pt, pfn); 607 page = xa_untag_pointer(entry); 608 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 609 return -ENOENT; 610 611 memcpy_to_page(page, 0, ptr, PAGE_SIZE); 612 613 ptr += PAGE_SIZE; 614 bounce->cpages++; 615 } 616 617 return 0; 618 } 619 620 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 621 { 622 struct dmirror_bounce bounce; 623 unsigned long start, end; 624 unsigned long size = cmd->npages << PAGE_SHIFT; 625 int ret; 626 627 start = cmd->addr; 628 end = start + size; 629 if (end < start) 630 return -EINVAL; 631 632 ret = dmirror_bounce_init(&bounce, start, size); 633 if (ret) 634 return ret; 635 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 636 bounce.size)) { 637 ret = -EFAULT; 638 goto fini; 639 } 640 641 while (1) { 642 mutex_lock(&dmirror->mutex); 643 ret = dmirror_do_write(dmirror, start, end, &bounce); 644 mutex_unlock(&dmirror->mutex); 645 if (ret != -ENOENT) 646 break; 647 648 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 649 ret = dmirror_fault(dmirror, start, end, true); 650 if (ret) 651 break; 652 cmd->faults++; 653 } 654 655 fini: 656 cmd->cpages = bounce.cpages; 657 dmirror_bounce_fini(&bounce); 658 return ret; 659 } 660 661 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 662 struct page **ppage, bool is_large) 663 { 664 struct dmirror_chunk *devmem; 665 struct resource *res = NULL; 666 unsigned long pfn; 667 unsigned long pfn_first; 668 unsigned long pfn_last; 669 void *ptr; 670 int ret = -ENOMEM; 671 672 devmem = kzalloc_obj(*devmem); 673 if (!devmem) 674 return ret; 675 676 switch (mdevice->zone_device_type) { 677 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 678 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 679 "hmm_dmirror"); 680 if (IS_ERR_OR_NULL(res)) 681 goto err_devmem; 682 devmem->pagemap.range.start = res->start; 683 devmem->pagemap.range.end = res->end; 684 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 685 break; 686 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 687 devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? 688 spm_addr_dev0 : 689 spm_addr_dev1; 690 devmem->pagemap.range.end = devmem->pagemap.range.start + 691 DEVMEM_CHUNK_SIZE - 1; 692 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 693 break; 694 default: 695 ret = -EINVAL; 696 goto err_devmem; 697 } 698 699 devmem->pagemap.nr_range = 1; 700 devmem->pagemap.ops = &dmirror_devmem_ops; 701 devmem->pagemap.owner = mdevice; 702 703 mutex_lock(&mdevice->devmem_lock); 704 705 if (mdevice->devmem_count == mdevice->devmem_capacity) { 706 struct dmirror_chunk **new_chunks; 707 unsigned int new_capacity; 708 709 new_capacity = mdevice->devmem_capacity + 710 DEVMEM_CHUNKS_RESERVE; 711 new_chunks = krealloc(mdevice->devmem_chunks, 712 sizeof(new_chunks[0]) * new_capacity, 713 GFP_KERNEL); 714 if (!new_chunks) 715 goto err_release; 716 mdevice->devmem_capacity = new_capacity; 717 mdevice->devmem_chunks = new_chunks; 718 } 719 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 720 if (IS_ERR_OR_NULL(ptr)) { 721 if (ptr) 722 ret = PTR_ERR(ptr); 723 else 724 ret = -EFAULT; 725 goto err_release; 726 } 727 728 devmem->mdevice = mdevice; 729 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 730 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 731 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 732 733 mutex_unlock(&mdevice->devmem_lock); 734 735 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 736 DEVMEM_CHUNK_SIZE / (1024 * 1024), 737 mdevice->devmem_count, 738 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 739 pfn_first, pfn_last); 740 741 spin_lock(&mdevice->lock); 742 for (pfn = pfn_first; pfn < pfn_last; ) { 743 struct page *page = pfn_to_page(pfn); 744 745 if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR) 746 && (pfn + HPAGE_PMD_NR <= pfn_last)) { 747 page->zone_device_data = mdevice->free_folios; 748 mdevice->free_folios = page_folio(page); 749 pfn += HPAGE_PMD_NR; 750 continue; 751 } 752 753 page->zone_device_data = mdevice->free_pages; 754 mdevice->free_pages = page; 755 pfn++; 756 } 757 758 ret = 0; 759 if (ppage) { 760 if (is_large) { 761 if (!mdevice->free_folios) { 762 ret = -ENOMEM; 763 goto err_unlock; 764 } 765 *ppage = folio_page(mdevice->free_folios, 0); 766 mdevice->free_folios = (*ppage)->zone_device_data; 767 mdevice->calloc += HPAGE_PMD_NR; 768 } else if (mdevice->free_pages) { 769 *ppage = mdevice->free_pages; 770 mdevice->free_pages = (*ppage)->zone_device_data; 771 mdevice->calloc++; 772 } else { 773 ret = -ENOMEM; 774 goto err_unlock; 775 } 776 } 777 err_unlock: 778 spin_unlock(&mdevice->lock); 779 780 return ret; 781 782 err_release: 783 mutex_unlock(&mdevice->devmem_lock); 784 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 785 release_mem_region(devmem->pagemap.range.start, 786 range_len(&devmem->pagemap.range)); 787 err_devmem: 788 kfree(devmem); 789 790 return ret; 791 } 792 793 static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, 794 bool is_large) 795 { 796 struct page *dpage = NULL; 797 struct page *rpage = NULL; 798 unsigned int order = is_large ? HPAGE_PMD_ORDER : 0; 799 struct dmirror_device *mdevice = dmirror->mdevice; 800 801 /* 802 * For ZONE_DEVICE private type, this is a fake device so we allocate 803 * real system memory to store our device memory. 804 * For ZONE_DEVICE coherent type we use the actual dpage to store the 805 * data and ignore rpage. 806 */ 807 if (dmirror_is_private_zone(mdevice)) { 808 rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0); 809 if (!rpage) 810 return NULL; 811 } 812 spin_lock(&mdevice->lock); 813 814 if (is_large && mdevice->free_folios) { 815 dpage = folio_page(mdevice->free_folios, 0); 816 mdevice->free_folios = dpage->zone_device_data; 817 mdevice->calloc += 1 << order; 818 spin_unlock(&mdevice->lock); 819 } else if (!is_large && mdevice->free_pages) { 820 dpage = mdevice->free_pages; 821 mdevice->free_pages = dpage->zone_device_data; 822 mdevice->calloc++; 823 spin_unlock(&mdevice->lock); 824 } else { 825 spin_unlock(&mdevice->lock); 826 if (dmirror_allocate_chunk(mdevice, &dpage, is_large)) 827 goto error; 828 } 829 830 zone_device_folio_init(page_folio(dpage), 831 page_pgmap(folio_page(page_folio(dpage), 0)), 832 order); 833 dpage->zone_device_data = rpage; 834 return dpage; 835 836 error: 837 if (rpage) 838 __free_pages(rpage, order); 839 return NULL; 840 } 841 842 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 843 struct dmirror *dmirror) 844 { 845 const unsigned long *src = args->src; 846 unsigned long *dst = args->dst; 847 unsigned long addr; 848 849 for (addr = args->start; addr < args->end; ) { 850 struct page *spage; 851 struct page *dpage; 852 struct page *rpage; 853 bool is_large = *src & MIGRATE_PFN_COMPOUND; 854 int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0; 855 unsigned long nr = 1; 856 857 if (!(*src & MIGRATE_PFN_MIGRATE)) 858 goto next; 859 860 /* 861 * Note that spage might be NULL which is OK since it is an 862 * unallocated pte_none() or read-only zero page. 863 */ 864 spage = migrate_pfn_to_page(*src); 865 if (WARN(spage && is_zone_device_page(spage), 866 "page already in device spage pfn: 0x%lx\n", 867 page_to_pfn(spage))) 868 goto next; 869 870 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { 871 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; 872 dpage = NULL; 873 } else 874 dpage = dmirror_devmem_alloc_page(dmirror, is_large); 875 876 if (!dpage) { 877 struct folio *folio; 878 unsigned long i; 879 unsigned long spfn = *src >> MIGRATE_PFN_SHIFT; 880 struct page *src_page; 881 882 if (!is_large) 883 goto next; 884 885 if (!spage && is_large) { 886 nr = HPAGE_PMD_NR; 887 } else { 888 folio = page_folio(spage); 889 nr = folio_nr_pages(folio); 890 } 891 892 for (i = 0; i < nr && addr < args->end; i++) { 893 dpage = dmirror_devmem_alloc_page(dmirror, false); 894 rpage = BACKING_PAGE(dpage); 895 rpage->zone_device_data = dmirror; 896 897 *dst = migrate_pfn(page_to_pfn(dpage)) | write; 898 src_page = pfn_to_page(spfn + i); 899 900 if (spage) 901 copy_highpage(rpage, src_page); 902 else 903 clear_highpage(rpage); 904 src++; 905 dst++; 906 addr += PAGE_SIZE; 907 } 908 continue; 909 } 910 911 rpage = BACKING_PAGE(dpage); 912 913 /* 914 * Normally, a device would use the page->zone_device_data to 915 * point to the mirror but here we use it to hold the page for 916 * the simulated device memory and that page holds the pointer 917 * to the mirror. 918 */ 919 rpage->zone_device_data = dmirror; 920 921 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 922 page_to_pfn(spage), page_to_pfn(dpage)); 923 924 *dst = migrate_pfn(page_to_pfn(dpage)) | write; 925 926 if (is_large) { 927 int i; 928 struct folio *folio = page_folio(dpage); 929 *dst |= MIGRATE_PFN_COMPOUND; 930 931 if (folio_test_large(folio)) { 932 for (i = 0; i < folio_nr_pages(folio); i++) { 933 struct page *dst_page = 934 pfn_to_page(page_to_pfn(rpage) + i); 935 struct page *src_page = 936 pfn_to_page(page_to_pfn(spage) + i); 937 938 if (spage) 939 copy_highpage(dst_page, src_page); 940 else 941 clear_highpage(dst_page); 942 src++; 943 dst++; 944 addr += PAGE_SIZE; 945 } 946 continue; 947 } 948 } 949 950 if (spage) 951 copy_highpage(rpage, spage); 952 else 953 clear_highpage(rpage); 954 955 next: 956 src++; 957 dst++; 958 addr += PAGE_SIZE; 959 } 960 } 961 962 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 963 unsigned long end) 964 { 965 unsigned long pfn; 966 967 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 968 void *entry; 969 970 entry = xa_load(&dmirror->pt, pfn); 971 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 972 return -EPERM; 973 } 974 975 return 0; 976 } 977 978 static int dmirror_atomic_map(unsigned long addr, struct page *page, 979 struct dmirror *dmirror) 980 { 981 void *entry; 982 983 /* Map the migrated pages into the device's page tables. */ 984 mutex_lock(&dmirror->mutex); 985 986 entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC); 987 entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); 988 if (xa_is_err(entry)) { 989 mutex_unlock(&dmirror->mutex); 990 return xa_err(entry); 991 } 992 993 mutex_unlock(&dmirror->mutex); 994 return 0; 995 } 996 997 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 998 struct dmirror *dmirror) 999 { 1000 unsigned long start = args->start; 1001 unsigned long end = args->end; 1002 const unsigned long *src = args->src; 1003 const unsigned long *dst = args->dst; 1004 unsigned long pfn; 1005 const unsigned long start_pfn = start >> PAGE_SHIFT; 1006 const unsigned long end_pfn = end >> PAGE_SHIFT; 1007 1008 /* Map the migrated pages into the device's page tables. */ 1009 mutex_lock(&dmirror->mutex); 1010 1011 for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) { 1012 struct page *dpage; 1013 void *entry; 1014 int nr, i; 1015 struct page *rpage; 1016 1017 if (!(*src & MIGRATE_PFN_MIGRATE)) 1018 continue; 1019 1020 dpage = migrate_pfn_to_page(*dst); 1021 if (!dpage) 1022 continue; 1023 1024 if (*dst & MIGRATE_PFN_COMPOUND) 1025 nr = folio_nr_pages(page_folio(dpage)); 1026 else 1027 nr = 1; 1028 1029 WARN_ON_ONCE(end_pfn < start_pfn + nr); 1030 1031 rpage = BACKING_PAGE(dpage); 1032 VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr); 1033 1034 for (i = 0; i < nr; i++) { 1035 entry = folio_page(page_folio(rpage), i); 1036 if (*dst & MIGRATE_PFN_WRITE) 1037 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 1038 entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC); 1039 if (xa_is_err(entry)) { 1040 mutex_unlock(&dmirror->mutex); 1041 return xa_err(entry); 1042 } 1043 } 1044 } 1045 1046 mutex_unlock(&dmirror->mutex); 1047 return 0; 1048 } 1049 1050 static int dmirror_exclusive(struct dmirror *dmirror, 1051 struct hmm_dmirror_cmd *cmd) 1052 { 1053 unsigned long start, end, addr; 1054 unsigned long size = cmd->npages << PAGE_SHIFT; 1055 struct mm_struct *mm = dmirror->notifier.mm; 1056 struct dmirror_bounce bounce; 1057 int ret = 0; 1058 1059 start = cmd->addr; 1060 end = start + size; 1061 if (end < start) 1062 return -EINVAL; 1063 1064 /* Since the mm is for the mirrored process, get a reference first. */ 1065 if (!mmget_not_zero(mm)) 1066 return -EINVAL; 1067 1068 mmap_read_lock(mm); 1069 for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { 1070 struct folio *folio; 1071 struct page *page; 1072 1073 page = make_device_exclusive(mm, addr, NULL, &folio); 1074 if (IS_ERR(page)) { 1075 ret = PTR_ERR(page); 1076 break; 1077 } 1078 1079 ret = dmirror_atomic_map(addr, page, dmirror); 1080 folio_unlock(folio); 1081 folio_put(folio); 1082 } 1083 mmap_read_unlock(mm); 1084 mmput(mm); 1085 1086 if (ret) 1087 return ret; 1088 1089 /* Return the migrated data for verification. */ 1090 ret = dmirror_bounce_init(&bounce, start, size); 1091 if (ret) 1092 return ret; 1093 mutex_lock(&dmirror->mutex); 1094 ret = dmirror_do_read(dmirror, start, end, &bounce); 1095 mutex_unlock(&dmirror->mutex); 1096 if (ret == 0) { 1097 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1098 bounce.size)) 1099 ret = -EFAULT; 1100 } 1101 1102 cmd->cpages = bounce.cpages; 1103 dmirror_bounce_fini(&bounce); 1104 return ret; 1105 } 1106 1107 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 1108 struct dmirror *dmirror) 1109 { 1110 const unsigned long *src = args->src; 1111 unsigned long *dst = args->dst; 1112 unsigned long start = args->start; 1113 unsigned long end = args->end; 1114 unsigned long addr; 1115 unsigned int order = 0; 1116 int i; 1117 1118 for (addr = start; addr < end; ) { 1119 struct page *dpage, *spage; 1120 1121 spage = migrate_pfn_to_page(*src); 1122 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) { 1123 addr += PAGE_SIZE; 1124 goto next; 1125 } 1126 1127 if (WARN_ON(!is_device_private_page(spage) && 1128 !is_device_coherent_page(spage))) { 1129 addr += PAGE_SIZE; 1130 goto next; 1131 } 1132 1133 spage = BACKING_PAGE(spage); 1134 order = folio_order(page_folio(spage)); 1135 if (order) 1136 *dst = MIGRATE_PFN_COMPOUND; 1137 if (*src & MIGRATE_PFN_WRITE) 1138 *dst |= MIGRATE_PFN_WRITE; 1139 1140 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { 1141 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; 1142 *dst &= ~MIGRATE_PFN_COMPOUND; 1143 dpage = NULL; 1144 } else if (order) { 1145 dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 1146 order, args->vma, addr), 0); 1147 } else { 1148 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 1149 } 1150 1151 if (!dpage && !order) 1152 return VM_FAULT_OOM; 1153 1154 if (dpage) { 1155 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", 1156 page_to_pfn(spage), page_to_pfn(dpage)); 1157 lock_page(dpage); 1158 *dst |= migrate_pfn(page_to_pfn(dpage)); 1159 } 1160 1161 for (i = 0; i < (1 << order); i++) { 1162 struct page *src_page; 1163 struct page *dst_page; 1164 1165 /* Try with smaller pages if large allocation fails */ 1166 if (!dpage && order) { 1167 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 1168 if (!dpage) { 1169 /* Unlock and free pages already allocated. */ 1170 while (i > 0) { 1171 struct page *fpage; 1172 1173 fpage = migrate_pfn_to_page(dst[--i]); 1174 unlock_page(fpage); 1175 __free_page(fpage); 1176 } 1177 /* Clear remaining dst entries to avoid 1178 * migrate_vma_pages/finalize() using 1179 * uninitialized values. 1180 */ 1181 while (i < (1 << order)) { 1182 dst[i] = 0; 1183 i++; 1184 } 1185 return VM_FAULT_OOM; 1186 } 1187 lock_page(dpage); 1188 dst[i] = migrate_pfn(page_to_pfn(dpage)); 1189 dst_page = pfn_to_page(page_to_pfn(dpage)); 1190 dpage = NULL; /* For the next iteration */ 1191 } else { 1192 dst_page = pfn_to_page(page_to_pfn(dpage) + i); 1193 } 1194 1195 src_page = pfn_to_page(page_to_pfn(spage) + i); 1196 1197 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 1198 addr += PAGE_SIZE; 1199 copy_highpage(dst_page, src_page); 1200 } 1201 next: 1202 src += 1 << order; 1203 dst += 1 << order; 1204 } 1205 return 0; 1206 } 1207 1208 static unsigned long 1209 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 1210 { 1211 unsigned long cpages = 0; 1212 unsigned long i; 1213 1214 for (i = 0; i < migrate->npages; i++) { 1215 if (migrate->src[i] & MIGRATE_PFN_VALID && 1216 migrate->src[i] & MIGRATE_PFN_MIGRATE) 1217 cpages++; 1218 } 1219 return cpages; 1220 } 1221 1222 static int dmirror_migrate_to_system(struct dmirror *dmirror, 1223 struct hmm_dmirror_cmd *cmd) 1224 { 1225 unsigned long start, end, addr; 1226 unsigned long size = cmd->npages << PAGE_SHIFT; 1227 struct mm_struct *mm = dmirror->notifier.mm; 1228 struct vm_area_struct *vma; 1229 struct migrate_vma args = { 0 }; 1230 unsigned long next; 1231 int ret; 1232 unsigned long *src_pfns; 1233 unsigned long *dst_pfns; 1234 1235 start = cmd->addr; 1236 end = start + size; 1237 if (end < start) 1238 return -EINVAL; 1239 1240 /* Since the mm is for the mirrored process, get a reference first. */ 1241 if (!mmget_not_zero(mm)) 1242 return -EINVAL; 1243 1244 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 1245 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 1246 1247 cmd->cpages = 0; 1248 mmap_read_lock(mm); 1249 for (addr = start; addr < end; addr = next) { 1250 vma = vma_lookup(mm, addr); 1251 if (!vma || !(vma->vm_flags & VM_READ)) { 1252 ret = -EINVAL; 1253 goto out; 1254 } 1255 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); 1256 if (next > vma->vm_end) 1257 next = vma->vm_end; 1258 1259 args.vma = vma; 1260 args.src = src_pfns; 1261 args.dst = dst_pfns; 1262 args.start = addr; 1263 args.end = next; 1264 args.pgmap_owner = dmirror->mdevice; 1265 args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND; 1266 1267 ret = migrate_vma_setup(&args); 1268 if (ret) 1269 goto out; 1270 1271 pr_debug("Migrating from device mem to sys mem\n"); 1272 if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { 1273 migrate_vma_finalize(&args); 1274 ret = -ENOMEM; 1275 goto out; 1276 } 1277 1278 migrate_vma_pages(&args); 1279 cmd->cpages += dmirror_successful_migrated_pages(&args); 1280 migrate_vma_finalize(&args); 1281 } 1282 out: 1283 mmap_read_unlock(mm); 1284 mmput(mm); 1285 kvfree(src_pfns); 1286 kvfree(dst_pfns); 1287 1288 return ret; 1289 } 1290 1291 static int dmirror_migrate_to_device(struct dmirror *dmirror, 1292 struct hmm_dmirror_cmd *cmd) 1293 { 1294 unsigned long start, end, addr; 1295 unsigned long size = cmd->npages << PAGE_SHIFT; 1296 struct mm_struct *mm = dmirror->notifier.mm; 1297 struct vm_area_struct *vma; 1298 struct dmirror_bounce bounce; 1299 struct migrate_vma args = { 0 }; 1300 unsigned long next; 1301 int ret; 1302 unsigned long *src_pfns = NULL; 1303 unsigned long *dst_pfns = NULL; 1304 1305 start = cmd->addr; 1306 end = start + size; 1307 if (end < start) 1308 return -EINVAL; 1309 1310 /* Since the mm is for the mirrored process, get a reference first. */ 1311 if (!mmget_not_zero(mm)) 1312 return -EINVAL; 1313 1314 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), 1315 GFP_KERNEL | __GFP_NOFAIL); 1316 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), 1317 GFP_KERNEL | __GFP_NOFAIL); 1318 1319 ret = 0; 1320 mmap_read_lock(mm); 1321 for (addr = start; addr < end; addr = next) { 1322 vma = vma_lookup(mm, addr); 1323 if (!vma || !(vma->vm_flags & VM_READ)) { 1324 ret = -EINVAL; 1325 goto out; 1326 } 1327 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); 1328 if (next > vma->vm_end) 1329 next = vma->vm_end; 1330 1331 args.vma = vma; 1332 args.src = src_pfns; 1333 args.dst = dst_pfns; 1334 args.start = addr; 1335 args.end = next; 1336 args.pgmap_owner = dmirror->mdevice; 1337 args.flags = MIGRATE_VMA_SELECT_SYSTEM | 1338 MIGRATE_VMA_SELECT_COMPOUND; 1339 ret = migrate_vma_setup(&args); 1340 if (ret) 1341 goto out; 1342 1343 pr_debug("Migrating from sys mem to device mem\n"); 1344 dmirror_migrate_alloc_and_copy(&args, dmirror); 1345 migrate_vma_pages(&args); 1346 dmirror_migrate_finalize_and_map(&args, dmirror); 1347 migrate_vma_finalize(&args); 1348 } 1349 mmap_read_unlock(mm); 1350 mmput(mm); 1351 1352 /* 1353 * Return the migrated data for verification. 1354 * Only for pages in device zone 1355 */ 1356 ret = dmirror_bounce_init(&bounce, start, size); 1357 if (ret) 1358 goto free_mem; 1359 mutex_lock(&dmirror->mutex); 1360 ret = dmirror_do_read(dmirror, start, end, &bounce); 1361 mutex_unlock(&dmirror->mutex); 1362 if (ret == 0) { 1363 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1364 bounce.size)) 1365 ret = -EFAULT; 1366 } 1367 cmd->cpages = bounce.cpages; 1368 dmirror_bounce_fini(&bounce); 1369 goto free_mem; 1370 1371 out: 1372 mmap_read_unlock(mm); 1373 mmput(mm); 1374 free_mem: 1375 kvfree(src_pfns); 1376 kvfree(dst_pfns); 1377 return ret; 1378 } 1379 1380 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1381 unsigned char *perm, unsigned long entry) 1382 { 1383 struct page *page; 1384 1385 if (entry & HMM_PFN_ERROR) { 1386 *perm = HMM_DMIRROR_PROT_ERROR; 1387 return; 1388 } 1389 if (!(entry & HMM_PFN_VALID)) { 1390 *perm = HMM_DMIRROR_PROT_NONE; 1391 return; 1392 } 1393 1394 page = hmm_pfn_to_page(entry); 1395 if (is_device_private_page(page)) { 1396 /* Is the page migrated to this device or some other? */ 1397 if (dmirror->mdevice == dmirror_page_to_device(page)) 1398 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1399 else 1400 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1401 } else if (is_device_coherent_page(page)) { 1402 /* Is the page migrated to this device or some other? */ 1403 if (dmirror->mdevice == dmirror_page_to_device(page)) 1404 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1405 else 1406 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1407 } else if (is_zero_pfn(page_to_pfn(page))) 1408 *perm = HMM_DMIRROR_PROT_ZERO; 1409 else 1410 *perm = HMM_DMIRROR_PROT_NONE; 1411 if (entry & HMM_PFN_WRITE) 1412 *perm |= HMM_DMIRROR_PROT_WRITE; 1413 else 1414 *perm |= HMM_DMIRROR_PROT_READ; 1415 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1416 *perm |= HMM_DMIRROR_PROT_PMD; 1417 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1418 *perm |= HMM_DMIRROR_PROT_PUD; 1419 } 1420 1421 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1422 const struct mmu_notifier_range *range, 1423 unsigned long cur_seq) 1424 { 1425 struct dmirror_interval *dmi = 1426 container_of(mni, struct dmirror_interval, notifier); 1427 struct dmirror *dmirror = dmi->dmirror; 1428 1429 if (mmu_notifier_range_blockable(range)) 1430 mutex_lock(&dmirror->mutex); 1431 else if (!mutex_trylock(&dmirror->mutex)) 1432 return false; 1433 1434 /* 1435 * Snapshots only need to set the sequence number since any 1436 * invalidation in the interval invalidates the whole snapshot. 1437 */ 1438 mmu_interval_set_seq(mni, cur_seq); 1439 1440 mutex_unlock(&dmirror->mutex); 1441 return true; 1442 } 1443 1444 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1445 .invalidate = dmirror_snapshot_invalidate, 1446 }; 1447 1448 static int dmirror_range_snapshot(struct dmirror *dmirror, 1449 struct hmm_range *range, 1450 unsigned char *perm) 1451 { 1452 struct mm_struct *mm = dmirror->notifier.mm; 1453 struct dmirror_interval notifier; 1454 unsigned long timeout = 1455 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1456 unsigned long i; 1457 unsigned long n; 1458 int ret = 0; 1459 1460 notifier.dmirror = dmirror; 1461 range->notifier = ¬ifier.notifier; 1462 1463 ret = mmu_interval_notifier_insert(range->notifier, mm, 1464 range->start, range->end - range->start, 1465 &dmirror_mrn_ops); 1466 if (ret) 1467 return ret; 1468 1469 while (true) { 1470 if (time_after(jiffies, timeout)) { 1471 ret = -EBUSY; 1472 goto out; 1473 } 1474 1475 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1476 1477 mmap_read_lock(mm); 1478 ret = hmm_range_fault(range); 1479 mmap_read_unlock(mm); 1480 if (ret) { 1481 if (ret == -EBUSY) 1482 continue; 1483 goto out; 1484 } 1485 1486 mutex_lock(&dmirror->mutex); 1487 if (mmu_interval_read_retry(range->notifier, 1488 range->notifier_seq)) { 1489 mutex_unlock(&dmirror->mutex); 1490 continue; 1491 } 1492 break; 1493 } 1494 1495 n = (range->end - range->start) >> PAGE_SHIFT; 1496 for (i = 0; i < n; i++) 1497 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1498 1499 mutex_unlock(&dmirror->mutex); 1500 out: 1501 mmu_interval_notifier_remove(range->notifier); 1502 return ret; 1503 } 1504 1505 static int dmirror_snapshot(struct dmirror *dmirror, 1506 struct hmm_dmirror_cmd *cmd) 1507 { 1508 struct mm_struct *mm = dmirror->notifier.mm; 1509 unsigned long start, end; 1510 unsigned long size = cmd->npages << PAGE_SHIFT; 1511 unsigned long addr; 1512 unsigned long next; 1513 unsigned long pfns[32]; 1514 unsigned char perm[32]; 1515 char __user *uptr; 1516 struct hmm_range range = { 1517 .hmm_pfns = pfns, 1518 .dev_private_owner = dmirror->mdevice, 1519 }; 1520 int ret = 0; 1521 1522 start = cmd->addr; 1523 end = start + size; 1524 if (end < start) 1525 return -EINVAL; 1526 1527 /* Since the mm is for the mirrored process, get a reference first. */ 1528 if (!mmget_not_zero(mm)) 1529 return -EINVAL; 1530 1531 /* 1532 * Register a temporary notifier to detect invalidations even if it 1533 * overlaps with other mmu_interval_notifiers. 1534 */ 1535 uptr = u64_to_user_ptr(cmd->ptr); 1536 for (addr = start; addr < end; addr = next) { 1537 unsigned long n; 1538 1539 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1540 range.start = addr; 1541 range.end = next; 1542 1543 ret = dmirror_range_snapshot(dmirror, &range, perm); 1544 if (ret) 1545 break; 1546 1547 n = (range.end - range.start) >> PAGE_SHIFT; 1548 if (copy_to_user(uptr, perm, n)) { 1549 ret = -EFAULT; 1550 break; 1551 } 1552 1553 cmd->cpages += n; 1554 uptr += n; 1555 } 1556 mmput(mm); 1557 1558 return ret; 1559 } 1560 1561 /* Removes free pages from the free list so they can't be re-allocated */ 1562 static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) 1563 { 1564 struct dmirror_device *mdevice = devmem->mdevice; 1565 struct page *page; 1566 struct folio *folio; 1567 1568 1569 for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio)) 1570 if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem) 1571 mdevice->free_folios = folio_zone_device_data(folio); 1572 for (page = mdevice->free_pages; page; page = page->zone_device_data) 1573 if (dmirror_page_to_chunk(page) == devmem) 1574 mdevice->free_pages = page->zone_device_data; 1575 } 1576 1577 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) 1578 { 1579 unsigned int i; 1580 1581 mutex_lock(&mdevice->devmem_lock); 1582 if (mdevice->devmem_chunks) { 1583 for (i = 0; i < mdevice->devmem_count; i++) { 1584 struct dmirror_chunk *devmem = 1585 mdevice->devmem_chunks[i]; 1586 1587 spin_lock(&mdevice->lock); 1588 devmem->remove = true; 1589 dmirror_remove_free_pages(devmem); 1590 spin_unlock(&mdevice->lock); 1591 1592 dmirror_device_evict_chunk(devmem); 1593 memunmap_pages(&devmem->pagemap); 1594 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1595 release_mem_region(devmem->pagemap.range.start, 1596 range_len(&devmem->pagemap.range)); 1597 kfree(devmem); 1598 } 1599 mdevice->devmem_count = 0; 1600 mdevice->devmem_capacity = 0; 1601 mdevice->free_pages = NULL; 1602 mdevice->free_folios = NULL; 1603 kfree(mdevice->devmem_chunks); 1604 mdevice->devmem_chunks = NULL; 1605 } 1606 mutex_unlock(&mdevice->devmem_lock); 1607 } 1608 1609 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1610 unsigned int command, 1611 unsigned long arg) 1612 { 1613 void __user *uarg = (void __user *)arg; 1614 struct hmm_dmirror_cmd cmd; 1615 struct dmirror *dmirror; 1616 int ret; 1617 1618 dmirror = filp->private_data; 1619 if (!dmirror) 1620 return -EINVAL; 1621 1622 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1623 return -EFAULT; 1624 1625 if (cmd.addr & ~PAGE_MASK) 1626 return -EINVAL; 1627 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1628 return -EINVAL; 1629 1630 cmd.cpages = 0; 1631 cmd.faults = 0; 1632 1633 switch (command) { 1634 case HMM_DMIRROR_READ: 1635 ret = dmirror_read(dmirror, &cmd); 1636 break; 1637 1638 case HMM_DMIRROR_WRITE: 1639 ret = dmirror_write(dmirror, &cmd); 1640 break; 1641 1642 case HMM_DMIRROR_MIGRATE_TO_DEV: 1643 ret = dmirror_migrate_to_device(dmirror, &cmd); 1644 break; 1645 1646 case HMM_DMIRROR_MIGRATE_TO_SYS: 1647 ret = dmirror_migrate_to_system(dmirror, &cmd); 1648 break; 1649 1650 case HMM_DMIRROR_EXCLUSIVE: 1651 ret = dmirror_exclusive(dmirror, &cmd); 1652 break; 1653 1654 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1655 ret = dmirror_check_atomic(dmirror, cmd.addr, 1656 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1657 break; 1658 1659 case HMM_DMIRROR_SNAPSHOT: 1660 ret = dmirror_snapshot(dmirror, &cmd); 1661 break; 1662 1663 case HMM_DMIRROR_RELEASE: 1664 dmirror_device_remove_chunks(dmirror->mdevice); 1665 ret = 0; 1666 break; 1667 case HMM_DMIRROR_FLAGS: 1668 dmirror->flags = cmd.npages; 1669 ret = 0; 1670 break; 1671 case HMM_DMIRROR_READ_UNLOCKED: 1672 ret = dmirror_read_unlocked(dmirror, &cmd, 0); 1673 break; 1674 default: 1675 return -EINVAL; 1676 } 1677 if (ret) 1678 return ret; 1679 1680 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1681 return -EFAULT; 1682 1683 return 0; 1684 } 1685 1686 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1687 { 1688 unsigned long addr; 1689 1690 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1691 struct page *page; 1692 int ret; 1693 1694 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1695 if (!page) 1696 return -ENOMEM; 1697 1698 ret = vm_insert_page(vma, addr, page); 1699 if (ret) { 1700 __free_page(page); 1701 return ret; 1702 } 1703 put_page(page); 1704 } 1705 1706 return 0; 1707 } 1708 1709 static const struct file_operations dmirror_fops = { 1710 .open = dmirror_fops_open, 1711 .release = dmirror_fops_release, 1712 .mmap = dmirror_fops_mmap, 1713 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1714 .llseek = default_llseek, 1715 .owner = THIS_MODULE, 1716 }; 1717 1718 static void dmirror_devmem_free(struct folio *folio) 1719 { 1720 struct page *page = &folio->page; 1721 struct page *rpage = BACKING_PAGE(page); 1722 struct dmirror_device *mdevice; 1723 struct folio *rfolio = page_folio(rpage); 1724 unsigned int order = folio_order(rfolio); 1725 1726 if (rpage != page) { 1727 if (order) 1728 __free_pages(rpage, order); 1729 else 1730 __free_page(rpage); 1731 rpage = NULL; 1732 } 1733 1734 mdevice = dmirror_page_to_device(page); 1735 spin_lock(&mdevice->lock); 1736 1737 /* Return page to our allocator if not freeing the chunk */ 1738 if (!dmirror_page_to_chunk(page)->remove) { 1739 mdevice->cfree += 1 << order; 1740 if (order) { 1741 page->zone_device_data = mdevice->free_folios; 1742 mdevice->free_folios = page_folio(page); 1743 } else { 1744 page->zone_device_data = mdevice->free_pages; 1745 mdevice->free_pages = page; 1746 } 1747 } 1748 spin_unlock(&mdevice->lock); 1749 } 1750 1751 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1752 { 1753 struct migrate_vma args = { 0 }; 1754 struct page *rpage; 1755 struct dmirror *dmirror; 1756 vm_fault_t ret = 0; 1757 unsigned int order, nr; 1758 1759 /* 1760 * Normally, a device would use the page->zone_device_data to point to 1761 * the mirror but here we use it to hold the page for the simulated 1762 * device memory and that page holds the pointer to the mirror. 1763 */ 1764 rpage = folio_zone_device_data(page_folio(vmf->page)); 1765 dmirror = rpage->zone_device_data; 1766 1767 /* FIXME demonstrate how we can adjust migrate range */ 1768 order = folio_order(page_folio(vmf->page)); 1769 nr = 1 << order; 1770 1771 /* 1772 * When folios are partially mapped, we can't rely on the folio 1773 * order of vmf->page as the folio might not be fully split yet 1774 */ 1775 if (vmf->pte) { 1776 order = 0; 1777 nr = 1; 1778 } 1779 1780 /* 1781 * Consider a per-cpu cache of src and dst pfns, but with 1782 * large number of cpus that might not scale well. 1783 */ 1784 args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order)); 1785 args.vma = vmf->vma; 1786 args.end = args.start + (PAGE_SIZE << order); 1787 1788 nr = (args.end - args.start) >> PAGE_SHIFT; 1789 args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); 1790 args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); 1791 args.pgmap_owner = dmirror->mdevice; 1792 args.flags = dmirror_select_device(dmirror); 1793 args.fault_page = vmf->page; 1794 1795 if (!args.src || !args.dst) { 1796 ret = VM_FAULT_OOM; 1797 goto err; 1798 } 1799 1800 if (order) 1801 args.flags |= MIGRATE_VMA_SELECT_COMPOUND; 1802 1803 /* 1804 * In practice migrate_vma_setup() should never fail unless the 1805 * test is wrong as it just tests some static VMA properties. 1806 */ 1807 if (migrate_vma_setup(&args)) { 1808 ret = VM_FAULT_SIGBUS; 1809 goto err; 1810 } 1811 1812 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1813 if (ret) { 1814 migrate_vma_finalize(&args); 1815 goto err; 1816 } 1817 migrate_vma_pages(&args); 1818 /* 1819 * No device finalize step is needed since 1820 * dmirror_devmem_fault_alloc_and_copy() will have already 1821 * invalidated the device page table. 1822 */ 1823 migrate_vma_finalize(&args); 1824 err: 1825 kfree(args.src); 1826 kfree(args.dst); 1827 return ret; 1828 } 1829 1830 static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail) 1831 { 1832 struct page *rpage = BACKING_PAGE(folio_page(head, 0)); 1833 struct page *rpage_tail; 1834 struct folio *rfolio; 1835 unsigned long offset = 0; 1836 1837 if (!rpage) { 1838 tail->page.zone_device_data = NULL; 1839 return; 1840 } 1841 1842 rfolio = page_folio(rpage); 1843 1844 if (tail == NULL) { 1845 folio_reset_order(rfolio); 1846 rfolio->mapping = NULL; 1847 folio_set_count(rfolio, 1); 1848 return; 1849 } 1850 1851 offset = folio_pfn(tail) - folio_pfn(head); 1852 1853 rpage_tail = folio_page(rfolio, offset); 1854 tail->page.zone_device_data = rpage_tail; 1855 rpage_tail->zone_device_data = rpage->zone_device_data; 1856 clear_compound_head(rpage_tail); 1857 rpage_tail->mapping = NULL; 1858 1859 folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping; 1860 tail->pgmap = head->pgmap; 1861 folio_set_count(page_folio(rpage_tail), 1); 1862 } 1863 1864 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1865 .folio_free = dmirror_devmem_free, 1866 .migrate_to_ram = dmirror_devmem_fault, 1867 .folio_split = dmirror_devmem_folio_split, 1868 }; 1869 1870 static void dmirror_device_release(struct device *dev) 1871 { 1872 struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device); 1873 1874 dmirror_device_remove_chunks(mdevice); 1875 } 1876 1877 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1878 { 1879 dev_t dev; 1880 int ret; 1881 1882 dev = MKDEV(MAJOR(dmirror_dev), id); 1883 mutex_init(&mdevice->devmem_lock); 1884 spin_lock_init(&mdevice->lock); 1885 1886 cdev_init(&mdevice->cdevice, &dmirror_fops); 1887 mdevice->cdevice.owner = THIS_MODULE; 1888 mdevice->device.release = dmirror_device_release; 1889 1890 device_initialize(&mdevice->device); 1891 mdevice->device.devt = dev; 1892 1893 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); 1894 if (ret) 1895 goto put_device; 1896 1897 /* Build a list of free ZONE_DEVICE struct pages */ 1898 ret = dmirror_allocate_chunk(mdevice, NULL, false); 1899 if (ret) 1900 goto put_device; 1901 1902 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); 1903 if (ret) 1904 goto put_device; 1905 1906 return 0; 1907 1908 put_device: 1909 put_device(&mdevice->device); 1910 return ret; 1911 } 1912 1913 static void dmirror_device_remove(struct dmirror_device *mdevice) 1914 { 1915 cdev_device_del(&mdevice->cdevice, &mdevice->device); 1916 put_device(&mdevice->device); 1917 } 1918 1919 static int __init hmm_dmirror_init(void) 1920 { 1921 int ret; 1922 int id = 0; 1923 int ndevices = 0; 1924 1925 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1926 "HMM_DMIRROR"); 1927 if (ret) 1928 goto err_unreg; 1929 1930 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1931 dmirror_devices[ndevices++].zone_device_type = 1932 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1933 dmirror_devices[ndevices++].zone_device_type = 1934 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1935 if (spm_addr_dev0 && spm_addr_dev1) { 1936 dmirror_devices[ndevices++].zone_device_type = 1937 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1938 dmirror_devices[ndevices++].zone_device_type = 1939 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1940 } 1941 for (id = 0; id < ndevices; id++) { 1942 ret = dmirror_device_init(dmirror_devices + id, id); 1943 if (ret) 1944 goto err_chrdev; 1945 } 1946 1947 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1948 return 0; 1949 1950 err_chrdev: 1951 while (--id >= 0) 1952 dmirror_device_remove(dmirror_devices + id); 1953 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1954 err_unreg: 1955 return ret; 1956 } 1957 1958 static void __exit hmm_dmirror_exit(void) 1959 { 1960 int id; 1961 1962 for (id = 0; id < DMIRROR_NDEVICES; id++) 1963 if (dmirror_devices[id].zone_device_type) 1964 dmirror_device_remove(dmirror_devices + id); 1965 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1966 } 1967 1968 module_init(hmm_dmirror_init); 1969 module_exit(hmm_dmirror_exit); 1970 MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module"); 1971 MODULE_LICENSE("GPL"); 1972