1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 __u64 flags; 96 }; 97 98 /* 99 * ZONE_DEVICE pages for migration and simulating device memory. 100 */ 101 struct dmirror_chunk { 102 struct dev_pagemap pagemap; 103 struct dmirror_device *mdevice; 104 bool remove; 105 }; 106 107 /* 108 * Per device data. 109 */ 110 struct dmirror_device { 111 struct cdev cdevice; 112 unsigned int zone_device_type; 113 struct device device; 114 115 unsigned int devmem_capacity; 116 unsigned int devmem_count; 117 struct dmirror_chunk **devmem_chunks; 118 struct mutex devmem_lock; /* protects the above */ 119 120 unsigned long calloc; 121 unsigned long cfree; 122 struct page *free_pages; 123 struct folio *free_folios; 124 spinlock_t lock; /* protects the above */ 125 }; 126 127 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 128 129 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 130 unsigned long addr, 131 unsigned long size) 132 { 133 bounce->addr = addr; 134 bounce->size = size; 135 bounce->cpages = 0; 136 bounce->ptr = vmalloc(size); 137 if (!bounce->ptr) 138 return -ENOMEM; 139 return 0; 140 } 141 142 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 143 { 144 return (mdevice->zone_device_type == 145 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE); 146 } 147 148 static enum migrate_vma_direction 149 dmirror_select_device(struct dmirror *dmirror) 150 { 151 return (dmirror->mdevice->zone_device_type == 152 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 153 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 154 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 155 } 156 157 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 158 { 159 vfree(bounce->ptr); 160 } 161 162 static int dmirror_fops_open(struct inode *inode, struct file *filp) 163 { 164 struct cdev *cdev = inode->i_cdev; 165 struct dmirror *dmirror; 166 int ret; 167 168 /* Mirror this process address space */ 169 dmirror = kzalloc_obj(*dmirror); 170 if (dmirror == NULL) 171 return -ENOMEM; 172 173 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 174 mutex_init(&dmirror->mutex); 175 xa_init(&dmirror->pt); 176 177 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 178 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 179 if (ret) { 180 kfree(dmirror); 181 return ret; 182 } 183 184 filp->private_data = dmirror; 185 return 0; 186 } 187 188 static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) 189 { 190 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; 191 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; 192 unsigned long npages = end_pfn - start_pfn + 1; 193 unsigned long i; 194 unsigned long *src_pfns; 195 unsigned long *dst_pfns; 196 unsigned int order = 0; 197 198 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 199 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 200 201 migrate_device_range(src_pfns, start_pfn, npages); 202 for (i = 0; i < npages; i++) { 203 struct page *dpage, *spage; 204 205 spage = migrate_pfn_to_page(src_pfns[i]); 206 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 207 continue; 208 209 if (WARN_ON(!is_device_private_page(spage) && 210 !is_device_coherent_page(spage))) 211 continue; 212 213 order = folio_order(page_folio(spage)); 214 spage = BACKING_PAGE(spage); 215 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { 216 dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, 217 order), 0); 218 } else { 219 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); 220 order = 0; 221 } 222 223 /* TODO Support splitting here */ 224 lock_page(dpage); 225 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 226 if (src_pfns[i] & MIGRATE_PFN_WRITE) 227 dst_pfns[i] |= MIGRATE_PFN_WRITE; 228 if (order) 229 dst_pfns[i] |= MIGRATE_PFN_COMPOUND; 230 folio_copy(page_folio(dpage), page_folio(spage)); 231 } 232 migrate_device_pages(src_pfns, dst_pfns, npages); 233 migrate_device_finalize(src_pfns, dst_pfns, npages); 234 kvfree(src_pfns); 235 kvfree(dst_pfns); 236 } 237 238 static int dmirror_fops_release(struct inode *inode, struct file *filp) 239 { 240 struct dmirror *dmirror = filp->private_data; 241 struct dmirror_device *mdevice = dmirror->mdevice; 242 int i; 243 244 mmu_interval_notifier_remove(&dmirror->notifier); 245 246 if (mdevice->devmem_chunks) { 247 for (i = 0; i < mdevice->devmem_count; i++) { 248 struct dmirror_chunk *devmem = 249 mdevice->devmem_chunks[i]; 250 251 dmirror_device_evict_chunk(devmem); 252 } 253 } 254 255 xa_destroy(&dmirror->pt); 256 kfree(dmirror); 257 return 0; 258 } 259 260 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) 261 { 262 return container_of(page_pgmap(page), struct dmirror_chunk, 263 pagemap); 264 } 265 266 static struct dmirror_device *dmirror_page_to_device(struct page *page) 267 268 { 269 return dmirror_page_to_chunk(page)->mdevice; 270 } 271 272 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 273 { 274 unsigned long *pfns = range->hmm_pfns; 275 unsigned long pfn; 276 277 for (pfn = (range->start >> PAGE_SHIFT); 278 pfn < (range->end >> PAGE_SHIFT); 279 pfn++, pfns++) { 280 struct page *page; 281 void *entry; 282 283 /* 284 * Since we asked for hmm_range_fault() to populate pages, 285 * it shouldn't return an error entry on success. 286 */ 287 WARN_ON(*pfns & HMM_PFN_ERROR); 288 WARN_ON(!(*pfns & HMM_PFN_VALID)); 289 290 page = hmm_pfn_to_page(*pfns); 291 WARN_ON(!page); 292 293 entry = page; 294 if (*pfns & HMM_PFN_WRITE) 295 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 296 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 297 return -EFAULT; 298 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 299 if (xa_is_err(entry)) 300 return xa_err(entry); 301 } 302 303 return 0; 304 } 305 306 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 307 unsigned long end) 308 { 309 unsigned long pfn; 310 void *entry; 311 312 /* 313 * The XArray doesn't hold references to pages since it relies on 314 * the mmu notifier to clear page pointers when they become stale. 315 * Therefore, it is OK to just clear the entry. 316 */ 317 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 318 end >> PAGE_SHIFT) 319 xa_erase(&dmirror->pt, pfn); 320 } 321 322 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 323 const struct mmu_notifier_range *range, 324 unsigned long cur_seq) 325 { 326 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 327 328 /* 329 * Ignore invalidation callbacks for device private pages since 330 * the invalidation is handled as part of the migration process. 331 */ 332 if (range->event == MMU_NOTIFY_MIGRATE && 333 range->owner == dmirror->mdevice) 334 return true; 335 336 if (mmu_notifier_range_blockable(range)) 337 mutex_lock(&dmirror->mutex); 338 else if (!mutex_trylock(&dmirror->mutex)) 339 return false; 340 341 mmu_interval_set_seq(mni, cur_seq); 342 dmirror_do_update(dmirror, range->start, range->end); 343 344 mutex_unlock(&dmirror->mutex); 345 return true; 346 } 347 348 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 349 .invalidate = dmirror_interval_invalidate, 350 }; 351 352 static int dmirror_range_fault(struct dmirror *dmirror, 353 struct hmm_range *range) 354 { 355 struct mm_struct *mm = dmirror->notifier.mm; 356 unsigned long timeout = 357 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 358 int ret; 359 360 while (true) { 361 if (time_after(jiffies, timeout)) { 362 ret = -EBUSY; 363 goto out; 364 } 365 366 range->notifier_seq = mmu_interval_read_begin(range->notifier); 367 mmap_read_lock(mm); 368 ret = hmm_range_fault(range); 369 mmap_read_unlock(mm); 370 if (ret) { 371 if (ret == -EBUSY) 372 continue; 373 goto out; 374 } 375 376 mutex_lock(&dmirror->mutex); 377 if (mmu_interval_read_retry(range->notifier, 378 range->notifier_seq)) { 379 mutex_unlock(&dmirror->mutex); 380 continue; 381 } 382 break; 383 } 384 385 ret = dmirror_do_fault(dmirror, range); 386 387 mutex_unlock(&dmirror->mutex); 388 out: 389 return ret; 390 } 391 392 static int dmirror_range_fault_unlocked(struct dmirror *dmirror, 393 struct hmm_range *range, 394 unsigned long timeout) 395 { 396 int ret; 397 398 while (true) { 399 ret = hmm_range_fault_unlocked_timeout(range, timeout); 400 if (ret) 401 goto out; 402 403 mutex_lock(&dmirror->mutex); 404 if (mmu_interval_read_retry(range->notifier, 405 range->notifier_seq)) { 406 mutex_unlock(&dmirror->mutex); 407 continue; 408 } 409 break; 410 } 411 412 ret = dmirror_do_fault(dmirror, range); 413 414 mutex_unlock(&dmirror->mutex); 415 out: 416 return ret; 417 } 418 419 static int dmirror_fault_unlocked(struct dmirror *dmirror, 420 unsigned long start, 421 unsigned long end, bool write, 422 unsigned long timeout) 423 { 424 struct mm_struct *mm = dmirror->notifier.mm; 425 unsigned long addr; 426 unsigned long pfns[32]; 427 struct hmm_range range = { 428 .notifier = &dmirror->notifier, 429 .hmm_pfns = pfns, 430 .pfn_flags_mask = 0, 431 .default_flags = 432 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 433 .dev_private_owner = dmirror->mdevice, 434 }; 435 int ret = 0; 436 437 if (!mmget_not_zero(mm)) 438 return -EFAULT; 439 440 for (addr = start; addr < end; addr = range.end) { 441 range.start = addr; 442 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 443 444 ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); 445 if (ret) 446 break; 447 } 448 449 mmput(mm); 450 return ret; 451 } 452 453 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 454 unsigned long end, bool write) 455 { 456 struct mm_struct *mm = dmirror->notifier.mm; 457 unsigned long addr; 458 unsigned long pfns[32]; 459 struct hmm_range range = { 460 .notifier = &dmirror->notifier, 461 .hmm_pfns = pfns, 462 .pfn_flags_mask = 0, 463 .default_flags = 464 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 465 .dev_private_owner = dmirror->mdevice, 466 }; 467 int ret = 0; 468 469 /* Since the mm is for the mirrored process, get a reference first. */ 470 if (!mmget_not_zero(mm)) 471 return -EFAULT; 472 473 for (addr = start; addr < end; addr = range.end) { 474 range.start = addr; 475 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 476 477 ret = dmirror_range_fault(dmirror, &range); 478 if (ret) 479 break; 480 } 481 482 mmput(mm); 483 return ret; 484 } 485 486 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 487 unsigned long end, struct dmirror_bounce *bounce) 488 { 489 unsigned long pfn; 490 void *ptr; 491 492 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 493 494 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 495 void *entry; 496 struct page *page; 497 498 entry = xa_load(&dmirror->pt, pfn); 499 page = xa_untag_pointer(entry); 500 if (!page) 501 return -ENOENT; 502 503 memcpy_from_page(ptr, page, 0, PAGE_SIZE); 504 505 ptr += PAGE_SIZE; 506 bounce->cpages++; 507 } 508 509 return 0; 510 } 511 512 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 513 { 514 struct dmirror_bounce bounce; 515 unsigned long start, end; 516 unsigned long size = cmd->npages << PAGE_SHIFT; 517 int ret; 518 519 start = cmd->addr; 520 end = start + size; 521 if (end < start) 522 return -EINVAL; 523 524 ret = dmirror_bounce_init(&bounce, start, size); 525 if (ret) 526 return ret; 527 528 while (1) { 529 mutex_lock(&dmirror->mutex); 530 ret = dmirror_do_read(dmirror, start, end, &bounce); 531 mutex_unlock(&dmirror->mutex); 532 if (ret != -ENOENT) 533 break; 534 535 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 536 ret = dmirror_fault(dmirror, start, end, false); 537 if (ret) 538 break; 539 cmd->faults++; 540 } 541 542 if (ret == 0) { 543 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 544 bounce.size)) 545 ret = -EFAULT; 546 } 547 cmd->cpages = bounce.cpages; 548 dmirror_bounce_fini(&bounce); 549 return ret; 550 } 551 552 static int dmirror_read_unlocked(struct dmirror *dmirror, 553 struct hmm_dmirror_cmd *cmd, 554 unsigned long timeout) 555 { 556 struct dmirror_bounce bounce; 557 unsigned long start, end; 558 unsigned long size = cmd->npages << PAGE_SHIFT; 559 int ret; 560 561 start = cmd->addr; 562 end = start + size; 563 if (end < start) 564 return -EINVAL; 565 566 ret = dmirror_bounce_init(&bounce, start, size); 567 if (ret) 568 return ret; 569 570 while (1) { 571 mutex_lock(&dmirror->mutex); 572 ret = dmirror_do_read(dmirror, start, end, &bounce); 573 mutex_unlock(&dmirror->mutex); 574 if (ret != -ENOENT) 575 break; 576 577 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 578 ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); 579 if (ret) 580 break; 581 cmd->faults++; 582 } 583 584 if (ret == 0) { 585 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 586 bounce.size)) 587 ret = -EFAULT; 588 } 589 cmd->cpages = bounce.cpages; 590 dmirror_bounce_fini(&bounce); 591 return ret; 592 } 593 594 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 595 unsigned long end, struct dmirror_bounce *bounce) 596 { 597 unsigned long pfn; 598 void *ptr; 599 600 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 601 602 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 603 void *entry; 604 struct page *page; 605 606 entry = xa_load(&dmirror->pt, pfn); 607 page = xa_untag_pointer(entry); 608 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 609 return -ENOENT; 610 611 memcpy_to_page(page, 0, ptr, PAGE_SIZE); 612 613 ptr += PAGE_SIZE; 614 bounce->cpages++; 615 } 616 617 return 0; 618 } 619 620 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 621 { 622 struct dmirror_bounce bounce; 623 unsigned long start, end; 624 unsigned long size = cmd->npages << PAGE_SHIFT; 625 int ret; 626 627 start = cmd->addr; 628 end = start + size; 629 if (end < start) 630 return -EINVAL; 631 632 ret = dmirror_bounce_init(&bounce, start, size); 633 if (ret) 634 return ret; 635 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 636 bounce.size)) { 637 ret = -EFAULT; 638 goto fini; 639 } 640 641 while (1) { 642 mutex_lock(&dmirror->mutex); 643 ret = dmirror_do_write(dmirror, start, end, &bounce); 644 mutex_unlock(&dmirror->mutex); 645 if (ret != -ENOENT) 646 break; 647 648 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 649 ret = dmirror_fault(dmirror, start, end, true); 650 if (ret) 651 break; 652 cmd->faults++; 653 } 654 655 fini: 656 cmd->cpages = bounce.cpages; 657 dmirror_bounce_fini(&bounce); 658 return ret; 659 } 660 661 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 662 struct page **ppage, bool is_large) 663 { 664 struct dmirror_chunk *devmem; 665 struct resource *res = NULL; 666 unsigned long pfn; 667 unsigned long pfn_first; 668 unsigned long pfn_last; 669 void *ptr; 670 int ret = -ENOMEM; 671 672 devmem = kzalloc_obj(*devmem); 673 if (!devmem) 674 return ret; 675 676 switch (mdevice->zone_device_type) { 677 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 678 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 679 "hmm_dmirror"); 680 if (IS_ERR_OR_NULL(res)) 681 goto err_devmem; 682 devmem->pagemap.range.start = res->start; 683 devmem->pagemap.range.end = res->end; 684 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 685 break; 686 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 687 devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? 688 spm_addr_dev0 : 689 spm_addr_dev1; 690 devmem->pagemap.range.end = devmem->pagemap.range.start + 691 DEVMEM_CHUNK_SIZE - 1; 692 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 693 break; 694 default: 695 ret = -EINVAL; 696 goto err_devmem; 697 } 698 699 devmem->pagemap.nr_range = 1; 700 devmem->pagemap.ops = &dmirror_devmem_ops; 701 devmem->pagemap.owner = mdevice; 702 703 mutex_lock(&mdevice->devmem_lock); 704 705 if (mdevice->devmem_count == mdevice->devmem_capacity) { 706 struct dmirror_chunk **new_chunks; 707 unsigned int new_capacity; 708 709 new_capacity = mdevice->devmem_capacity + 710 DEVMEM_CHUNKS_RESERVE; 711 new_chunks = krealloc(mdevice->devmem_chunks, 712 sizeof(new_chunks[0]) * new_capacity, 713 GFP_KERNEL); 714 if (!new_chunks) 715 goto err_release; 716 mdevice->devmem_capacity = new_capacity; 717 mdevice->devmem_chunks = new_chunks; 718 } 719 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 720 if (IS_ERR_OR_NULL(ptr)) { 721 if (ptr) 722 ret = PTR_ERR(ptr); 723 else 724 ret = -EFAULT; 725 goto err_release; 726 } 727 728 devmem->mdevice = mdevice; 729 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 730 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 731 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 732 733 mutex_unlock(&mdevice->devmem_lock); 734 735 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 736 DEVMEM_CHUNK_SIZE / (1024 * 1024), 737 mdevice->devmem_count, 738 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 739 pfn_first, pfn_last); 740 741 spin_lock(&mdevice->lock); 742 for (pfn = pfn_first; pfn < pfn_last; ) { 743 struct page *page = pfn_to_page(pfn); 744 745 if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR) 746 && (pfn + HPAGE_PMD_NR <= pfn_last)) { 747 page->zone_device_data = mdevice->free_folios; 748 mdevice->free_folios = page_folio(page); 749 pfn += HPAGE_PMD_NR; 750 continue; 751 } 752 753 page->zone_device_data = mdevice->free_pages; 754 mdevice->free_pages = page; 755 pfn++; 756 } 757 758 ret = 0; 759 if (ppage) { 760 if (is_large) { 761 if (!mdevice->free_folios) { 762 ret = -ENOMEM; 763 goto err_unlock; 764 } 765 *ppage = folio_page(mdevice->free_folios, 0); 766 mdevice->free_folios = (*ppage)->zone_device_data; 767 mdevice->calloc += HPAGE_PMD_NR; 768 } else if (mdevice->free_pages) { 769 *ppage = mdevice->free_pages; 770 mdevice->free_pages = (*ppage)->zone_device_data; 771 mdevice->calloc++; 772 } else { 773 ret = -ENOMEM; 774 goto err_unlock; 775 } 776 } 777 err_unlock: 778 spin_unlock(&mdevice->lock); 779 780 return ret; 781 782 err_release: 783 mutex_unlock(&mdevice->devmem_lock); 784 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 785 release_mem_region(devmem->pagemap.range.start, 786 range_len(&devmem->pagemap.range)); 787 err_devmem: 788 kfree(devmem); 789 790 return ret; 791 } 792 793 static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, 794 bool is_large) 795 { 796 struct page *dpage = NULL; 797 struct page *rpage = NULL; 798 unsigned int order = is_large ? HPAGE_PMD_ORDER : 0; 799 struct dmirror_device *mdevice = dmirror->mdevice; 800 801 /* 802 * For ZONE_DEVICE private type, this is a fake device so we allocate 803 * real system memory to store our device memory. 804 * For ZONE_DEVICE coherent type we use the actual dpage to store the 805 * data and ignore rpage. 806 */ 807 if (dmirror_is_private_zone(mdevice)) { 808 rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0); 809 if (!rpage) 810 return NULL; 811 } 812 spin_lock(&mdevice->lock); 813 814 if (is_large && mdevice->free_folios) { 815 dpage = folio_page(mdevice->free_folios, 0); 816 mdevice->free_folios = dpage->zone_device_data; 817 mdevice->calloc += 1 << order; 818 spin_unlock(&mdevice->lock); 819 } else if (!is_large && mdevice->free_pages) { 820 dpage = mdevice->free_pages; 821 mdevice->free_pages = dpage->zone_device_data; 822 mdevice->calloc++; 823 spin_unlock(&mdevice->lock); 824 } else { 825 spin_unlock(&mdevice->lock); 826 if (dmirror_allocate_chunk(mdevice, &dpage, is_large)) 827 goto error; 828 } 829 830 zone_device_folio_init(page_folio(dpage), 831 page_pgmap(folio_page(page_folio(dpage), 0)), 832 order); 833 dpage->zone_device_data = rpage; 834 return dpage; 835 836 error: 837 if (rpage) 838 __free_pages(rpage, order); 839 return NULL; 840 } 841 842 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 843 struct dmirror *dmirror) 844 { 845 const unsigned long *src = args->src; 846 unsigned long *dst = args->dst; 847 unsigned long addr; 848 849 for (addr = args->start; addr < args->end; ) { 850 struct page *spage; 851 struct page *dpage; 852 struct page *rpage; 853 bool is_large = *src & MIGRATE_PFN_COMPOUND; 854 int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0; 855 unsigned long nr = 1; 856 857 if (!(*src & MIGRATE_PFN_MIGRATE)) 858 goto next; 859 860 /* 861 * Note that spage might be NULL which is OK since it is an 862 * unallocated pte_none() or read-only zero page. 863 */ 864 spage = migrate_pfn_to_page(*src); 865 if (WARN(spage && is_zone_device_page(spage), 866 "page already in device spage pfn: 0x%lx\n", 867 page_to_pfn(spage))) 868 goto next; 869 870 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { 871 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; 872 dpage = NULL; 873 } else 874 dpage = dmirror_devmem_alloc_page(dmirror, is_large); 875 876 if (!dpage) { 877 struct folio *folio; 878 unsigned long i; 879 unsigned long spfn = *src >> MIGRATE_PFN_SHIFT; 880 struct page *src_page; 881 882 if (!is_large) 883 goto next; 884 885 if (!spage && is_large) { 886 nr = HPAGE_PMD_NR; 887 } else { 888 folio = page_folio(spage); 889 nr = folio_nr_pages(folio); 890 } 891 892 for (i = 0; i < nr && addr < args->end; i++) { 893 dpage = dmirror_devmem_alloc_page(dmirror, false); 894 rpage = BACKING_PAGE(dpage); 895 rpage->zone_device_data = dmirror; 896 897 *dst = migrate_pfn(page_to_pfn(dpage)) | write; 898 src_page = pfn_to_page(spfn + i); 899 900 if (spage) 901 copy_highpage(rpage, src_page); 902 else 903 clear_highpage(rpage); 904 src++; 905 dst++; 906 addr += PAGE_SIZE; 907 } 908 continue; 909 } 910 911 rpage = BACKING_PAGE(dpage); 912 913 /* 914 * Normally, a device would use the page->zone_device_data to 915 * point to the mirror but here we use it to hold the page for 916 * the simulated device memory and that page holds the pointer 917 * to the mirror. 918 */ 919 rpage->zone_device_data = dmirror; 920 921 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 922 page_to_pfn(spage), page_to_pfn(dpage)); 923 924 *dst = migrate_pfn(page_to_pfn(dpage)) | write; 925 926 if (is_large) { 927 int i; 928 struct folio *folio = page_folio(dpage); 929 *dst |= MIGRATE_PFN_COMPOUND; 930 931 if (folio_test_large(folio)) { 932 for (i = 0; i < folio_nr_pages(folio); i++) { 933 struct page *dst_page = 934 pfn_to_page(page_to_pfn(rpage) + i); 935 struct page *src_page = 936 pfn_to_page(page_to_pfn(spage) + i); 937 938 if (spage) 939 copy_highpage(dst_page, src_page); 940 else 941 clear_highpage(dst_page); 942 src++; 943 dst++; 944 addr += PAGE_SIZE; 945 } 946 continue; 947 } 948 } 949 950 if (spage) 951 copy_highpage(rpage, spage); 952 else 953 clear_highpage(rpage); 954 955 next: 956 src++; 957 dst++; 958 addr += PAGE_SIZE; 959 } 960 } 961 962 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 963 unsigned long end) 964 { 965 unsigned long pfn; 966 967 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 968 void *entry; 969 970 entry = xa_load(&dmirror->pt, pfn); 971 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 972 return -EPERM; 973 } 974 975 return 0; 976 } 977 978 static int dmirror_atomic_map(unsigned long addr, struct page *page, 979 struct dmirror *dmirror) 980 { 981 void *entry; 982 983 /* Map the migrated pages into the device's page tables. */ 984 mutex_lock(&dmirror->mutex); 985 986 entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC); 987 entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC); 988 if (xa_is_err(entry)) { 989 mutex_unlock(&dmirror->mutex); 990 return xa_err(entry); 991 } 992 993 mutex_unlock(&dmirror->mutex); 994 return 0; 995 } 996 997 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 998 struct dmirror *dmirror) 999 { 1000 unsigned long start = args->start; 1001 unsigned long end = args->end; 1002 const unsigned long *src = args->src; 1003 const unsigned long *dst = args->dst; 1004 unsigned long pfn; 1005 const unsigned long start_pfn = start >> PAGE_SHIFT; 1006 const unsigned long end_pfn = end >> PAGE_SHIFT; 1007 1008 /* Map the migrated pages into the device's page tables. */ 1009 mutex_lock(&dmirror->mutex); 1010 1011 for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) { 1012 struct page *dpage; 1013 void *entry; 1014 int nr, i; 1015 struct page *rpage; 1016 1017 if (!(*src & MIGRATE_PFN_MIGRATE)) 1018 continue; 1019 1020 dpage = migrate_pfn_to_page(*dst); 1021 if (!dpage) 1022 continue; 1023 1024 if (*dst & MIGRATE_PFN_COMPOUND) 1025 nr = folio_nr_pages(page_folio(dpage)); 1026 else 1027 nr = 1; 1028 1029 WARN_ON_ONCE(end_pfn < start_pfn + nr); 1030 1031 rpage = BACKING_PAGE(dpage); 1032 VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr); 1033 1034 for (i = 0; i < nr; i++) { 1035 entry = folio_page(page_folio(rpage), i); 1036 if (*dst & MIGRATE_PFN_WRITE) 1037 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 1038 entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC); 1039 if (xa_is_err(entry)) { 1040 mutex_unlock(&dmirror->mutex); 1041 return xa_err(entry); 1042 } 1043 } 1044 } 1045 1046 mutex_unlock(&dmirror->mutex); 1047 return 0; 1048 } 1049 1050 static int dmirror_exclusive(struct dmirror *dmirror, 1051 struct hmm_dmirror_cmd *cmd) 1052 { 1053 unsigned long start, end, addr; 1054 unsigned long size = cmd->npages << PAGE_SHIFT; 1055 struct mm_struct *mm = dmirror->notifier.mm; 1056 struct dmirror_bounce bounce; 1057 int ret = 0; 1058 1059 start = cmd->addr; 1060 end = start + size; 1061 if (end < start) 1062 return -EINVAL; 1063 1064 /* Since the mm is for the mirrored process, get a reference first. */ 1065 if (!mmget_not_zero(mm)) 1066 return -EINVAL; 1067 1068 mmap_read_lock(mm); 1069 for (addr = start; !ret && addr < end; addr += PAGE_SIZE) { 1070 struct folio *folio; 1071 struct page *page; 1072 1073 page = make_device_exclusive(mm, addr, NULL, &folio); 1074 if (IS_ERR(page)) { 1075 ret = PTR_ERR(page); 1076 break; 1077 } 1078 1079 ret = dmirror_atomic_map(addr, page, dmirror); 1080 folio_unlock(folio); 1081 folio_put(folio); 1082 } 1083 mmap_read_unlock(mm); 1084 mmput(mm); 1085 1086 if (ret) 1087 return ret; 1088 1089 /* Return the migrated data for verification. */ 1090 ret = dmirror_bounce_init(&bounce, start, size); 1091 if (ret) 1092 return ret; 1093 mutex_lock(&dmirror->mutex); 1094 ret = dmirror_do_read(dmirror, start, end, &bounce); 1095 mutex_unlock(&dmirror->mutex); 1096 if (ret == 0) { 1097 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1098 bounce.size)) 1099 ret = -EFAULT; 1100 } 1101 1102 cmd->cpages = bounce.cpages; 1103 dmirror_bounce_fini(&bounce); 1104 return ret; 1105 } 1106 1107 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 1108 struct dmirror *dmirror) 1109 { 1110 const unsigned long *src = args->src; 1111 unsigned long *dst = args->dst; 1112 unsigned long start = args->start; 1113 unsigned long end = args->end; 1114 unsigned long addr; 1115 unsigned int order = 0; 1116 int i; 1117 1118 for (addr = start; addr < end; ) { 1119 struct page *dpage, *spage; 1120 1121 spage = migrate_pfn_to_page(*src); 1122 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) { 1123 addr += PAGE_SIZE; 1124 goto next; 1125 } 1126 1127 if (WARN_ON(!is_device_private_page(spage) && 1128 !is_device_coherent_page(spage))) { 1129 addr += PAGE_SIZE; 1130 goto next; 1131 } 1132 1133 spage = BACKING_PAGE(spage); 1134 order = folio_order(page_folio(spage)); 1135 if (order) 1136 *dst = MIGRATE_PFN_COMPOUND; 1137 if (*src & MIGRATE_PFN_WRITE) 1138 *dst |= MIGRATE_PFN_WRITE; 1139 1140 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) { 1141 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC; 1142 *dst &= ~MIGRATE_PFN_COMPOUND; 1143 dpage = NULL; 1144 } else if (order) { 1145 dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 1146 order, args->vma, addr), 0); 1147 } else { 1148 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 1149 } 1150 1151 if (!dpage && !order) 1152 return VM_FAULT_OOM; 1153 1154 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 1155 page_to_pfn(spage), page_to_pfn(dpage)); 1156 1157 if (dpage) { 1158 lock_page(dpage); 1159 *dst |= migrate_pfn(page_to_pfn(dpage)); 1160 } 1161 1162 for (i = 0; i < (1 << order); i++) { 1163 struct page *src_page; 1164 struct page *dst_page; 1165 1166 /* Try with smaller pages if large allocation fails */ 1167 if (!dpage && order) { 1168 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 1169 if (!dpage) { 1170 /* Unlock and free pages already allocated. */ 1171 while (i > 0) { 1172 struct page *fpage; 1173 1174 fpage = migrate_pfn_to_page(dst[--i]); 1175 unlock_page(fpage); 1176 __free_page(fpage); 1177 } 1178 /* Clear remaining dst entries to avoid 1179 * migrate_vma_pages/finalize() using 1180 * uninitialized values. 1181 */ 1182 while (i < (1 << order)) { 1183 dst[i] = 0; 1184 i++; 1185 } 1186 return VM_FAULT_OOM; 1187 } 1188 lock_page(dpage); 1189 dst[i] = migrate_pfn(page_to_pfn(dpage)); 1190 dst_page = pfn_to_page(page_to_pfn(dpage)); 1191 dpage = NULL; /* For the next iteration */ 1192 } else { 1193 dst_page = pfn_to_page(page_to_pfn(dpage) + i); 1194 } 1195 1196 src_page = pfn_to_page(page_to_pfn(spage) + i); 1197 1198 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 1199 addr += PAGE_SIZE; 1200 copy_highpage(dst_page, src_page); 1201 } 1202 next: 1203 src += 1 << order; 1204 dst += 1 << order; 1205 } 1206 return 0; 1207 } 1208 1209 static unsigned long 1210 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 1211 { 1212 unsigned long cpages = 0; 1213 unsigned long i; 1214 1215 for (i = 0; i < migrate->npages; i++) { 1216 if (migrate->src[i] & MIGRATE_PFN_VALID && 1217 migrate->src[i] & MIGRATE_PFN_MIGRATE) 1218 cpages++; 1219 } 1220 return cpages; 1221 } 1222 1223 static int dmirror_migrate_to_system(struct dmirror *dmirror, 1224 struct hmm_dmirror_cmd *cmd) 1225 { 1226 unsigned long start, end, addr; 1227 unsigned long size = cmd->npages << PAGE_SHIFT; 1228 struct mm_struct *mm = dmirror->notifier.mm; 1229 struct vm_area_struct *vma; 1230 struct migrate_vma args = { 0 }; 1231 unsigned long next; 1232 int ret; 1233 unsigned long *src_pfns; 1234 unsigned long *dst_pfns; 1235 1236 start = cmd->addr; 1237 end = start + size; 1238 if (end < start) 1239 return -EINVAL; 1240 1241 /* Since the mm is for the mirrored process, get a reference first. */ 1242 if (!mmget_not_zero(mm)) 1243 return -EINVAL; 1244 1245 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 1246 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 1247 1248 cmd->cpages = 0; 1249 mmap_read_lock(mm); 1250 for (addr = start; addr < end; addr = next) { 1251 vma = vma_lookup(mm, addr); 1252 if (!vma || !(vma->vm_flags & VM_READ)) { 1253 ret = -EINVAL; 1254 goto out; 1255 } 1256 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); 1257 if (next > vma->vm_end) 1258 next = vma->vm_end; 1259 1260 args.vma = vma; 1261 args.src = src_pfns; 1262 args.dst = dst_pfns; 1263 args.start = addr; 1264 args.end = next; 1265 args.pgmap_owner = dmirror->mdevice; 1266 args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND; 1267 1268 ret = migrate_vma_setup(&args); 1269 if (ret) 1270 goto out; 1271 1272 pr_debug("Migrating from device mem to sys mem\n"); 1273 if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { 1274 migrate_vma_finalize(&args); 1275 ret = -ENOMEM; 1276 goto out; 1277 } 1278 1279 migrate_vma_pages(&args); 1280 cmd->cpages += dmirror_successful_migrated_pages(&args); 1281 migrate_vma_finalize(&args); 1282 } 1283 out: 1284 mmap_read_unlock(mm); 1285 mmput(mm); 1286 kvfree(src_pfns); 1287 kvfree(dst_pfns); 1288 1289 return ret; 1290 } 1291 1292 static int dmirror_migrate_to_device(struct dmirror *dmirror, 1293 struct hmm_dmirror_cmd *cmd) 1294 { 1295 unsigned long start, end, addr; 1296 unsigned long size = cmd->npages << PAGE_SHIFT; 1297 struct mm_struct *mm = dmirror->notifier.mm; 1298 struct vm_area_struct *vma; 1299 struct dmirror_bounce bounce; 1300 struct migrate_vma args = { 0 }; 1301 unsigned long next; 1302 int ret; 1303 unsigned long *src_pfns = NULL; 1304 unsigned long *dst_pfns = NULL; 1305 1306 start = cmd->addr; 1307 end = start + size; 1308 if (end < start) 1309 return -EINVAL; 1310 1311 /* Since the mm is for the mirrored process, get a reference first. */ 1312 if (!mmget_not_zero(mm)) 1313 return -EINVAL; 1314 1315 ret = -ENOMEM; 1316 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), 1317 GFP_KERNEL | __GFP_NOFAIL); 1318 if (!src_pfns) 1319 goto free_mem; 1320 1321 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), 1322 GFP_KERNEL | __GFP_NOFAIL); 1323 if (!dst_pfns) 1324 goto free_mem; 1325 1326 ret = 0; 1327 mmap_read_lock(mm); 1328 for (addr = start; addr < end; addr = next) { 1329 vma = vma_lookup(mm, addr); 1330 if (!vma || !(vma->vm_flags & VM_READ)) { 1331 ret = -EINVAL; 1332 goto out; 1333 } 1334 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT)); 1335 if (next > vma->vm_end) 1336 next = vma->vm_end; 1337 1338 args.vma = vma; 1339 args.src = src_pfns; 1340 args.dst = dst_pfns; 1341 args.start = addr; 1342 args.end = next; 1343 args.pgmap_owner = dmirror->mdevice; 1344 args.flags = MIGRATE_VMA_SELECT_SYSTEM | 1345 MIGRATE_VMA_SELECT_COMPOUND; 1346 ret = migrate_vma_setup(&args); 1347 if (ret) 1348 goto out; 1349 1350 pr_debug("Migrating from sys mem to device mem\n"); 1351 dmirror_migrate_alloc_and_copy(&args, dmirror); 1352 migrate_vma_pages(&args); 1353 dmirror_migrate_finalize_and_map(&args, dmirror); 1354 migrate_vma_finalize(&args); 1355 } 1356 mmap_read_unlock(mm); 1357 mmput(mm); 1358 1359 /* 1360 * Return the migrated data for verification. 1361 * Only for pages in device zone 1362 */ 1363 ret = dmirror_bounce_init(&bounce, start, size); 1364 if (ret) 1365 goto free_mem; 1366 mutex_lock(&dmirror->mutex); 1367 ret = dmirror_do_read(dmirror, start, end, &bounce); 1368 mutex_unlock(&dmirror->mutex); 1369 if (ret == 0) { 1370 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1371 bounce.size)) 1372 ret = -EFAULT; 1373 } 1374 cmd->cpages = bounce.cpages; 1375 dmirror_bounce_fini(&bounce); 1376 goto free_mem; 1377 1378 out: 1379 mmap_read_unlock(mm); 1380 mmput(mm); 1381 free_mem: 1382 kvfree(src_pfns); 1383 kvfree(dst_pfns); 1384 return ret; 1385 } 1386 1387 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1388 unsigned char *perm, unsigned long entry) 1389 { 1390 struct page *page; 1391 1392 if (entry & HMM_PFN_ERROR) { 1393 *perm = HMM_DMIRROR_PROT_ERROR; 1394 return; 1395 } 1396 if (!(entry & HMM_PFN_VALID)) { 1397 *perm = HMM_DMIRROR_PROT_NONE; 1398 return; 1399 } 1400 1401 page = hmm_pfn_to_page(entry); 1402 if (is_device_private_page(page)) { 1403 /* Is the page migrated to this device or some other? */ 1404 if (dmirror->mdevice == dmirror_page_to_device(page)) 1405 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1406 else 1407 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1408 } else if (is_device_coherent_page(page)) { 1409 /* Is the page migrated to this device or some other? */ 1410 if (dmirror->mdevice == dmirror_page_to_device(page)) 1411 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1412 else 1413 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1414 } else if (is_zero_pfn(page_to_pfn(page))) 1415 *perm = HMM_DMIRROR_PROT_ZERO; 1416 else 1417 *perm = HMM_DMIRROR_PROT_NONE; 1418 if (entry & HMM_PFN_WRITE) 1419 *perm |= HMM_DMIRROR_PROT_WRITE; 1420 else 1421 *perm |= HMM_DMIRROR_PROT_READ; 1422 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1423 *perm |= HMM_DMIRROR_PROT_PMD; 1424 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1425 *perm |= HMM_DMIRROR_PROT_PUD; 1426 } 1427 1428 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1429 const struct mmu_notifier_range *range, 1430 unsigned long cur_seq) 1431 { 1432 struct dmirror_interval *dmi = 1433 container_of(mni, struct dmirror_interval, notifier); 1434 struct dmirror *dmirror = dmi->dmirror; 1435 1436 if (mmu_notifier_range_blockable(range)) 1437 mutex_lock(&dmirror->mutex); 1438 else if (!mutex_trylock(&dmirror->mutex)) 1439 return false; 1440 1441 /* 1442 * Snapshots only need to set the sequence number since any 1443 * invalidation in the interval invalidates the whole snapshot. 1444 */ 1445 mmu_interval_set_seq(mni, cur_seq); 1446 1447 mutex_unlock(&dmirror->mutex); 1448 return true; 1449 } 1450 1451 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1452 .invalidate = dmirror_snapshot_invalidate, 1453 }; 1454 1455 static int dmirror_range_snapshot(struct dmirror *dmirror, 1456 struct hmm_range *range, 1457 unsigned char *perm) 1458 { 1459 struct mm_struct *mm = dmirror->notifier.mm; 1460 struct dmirror_interval notifier; 1461 unsigned long timeout = 1462 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1463 unsigned long i; 1464 unsigned long n; 1465 int ret = 0; 1466 1467 notifier.dmirror = dmirror; 1468 range->notifier = ¬ifier.notifier; 1469 1470 ret = mmu_interval_notifier_insert(range->notifier, mm, 1471 range->start, range->end - range->start, 1472 &dmirror_mrn_ops); 1473 if (ret) 1474 return ret; 1475 1476 while (true) { 1477 if (time_after(jiffies, timeout)) { 1478 ret = -EBUSY; 1479 goto out; 1480 } 1481 1482 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1483 1484 mmap_read_lock(mm); 1485 ret = hmm_range_fault(range); 1486 mmap_read_unlock(mm); 1487 if (ret) { 1488 if (ret == -EBUSY) 1489 continue; 1490 goto out; 1491 } 1492 1493 mutex_lock(&dmirror->mutex); 1494 if (mmu_interval_read_retry(range->notifier, 1495 range->notifier_seq)) { 1496 mutex_unlock(&dmirror->mutex); 1497 continue; 1498 } 1499 break; 1500 } 1501 1502 n = (range->end - range->start) >> PAGE_SHIFT; 1503 for (i = 0; i < n; i++) 1504 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1505 1506 mutex_unlock(&dmirror->mutex); 1507 out: 1508 mmu_interval_notifier_remove(range->notifier); 1509 return ret; 1510 } 1511 1512 static int dmirror_snapshot(struct dmirror *dmirror, 1513 struct hmm_dmirror_cmd *cmd) 1514 { 1515 struct mm_struct *mm = dmirror->notifier.mm; 1516 unsigned long start, end; 1517 unsigned long size = cmd->npages << PAGE_SHIFT; 1518 unsigned long addr; 1519 unsigned long next; 1520 unsigned long pfns[32]; 1521 unsigned char perm[32]; 1522 char __user *uptr; 1523 struct hmm_range range = { 1524 .hmm_pfns = pfns, 1525 .dev_private_owner = dmirror->mdevice, 1526 }; 1527 int ret = 0; 1528 1529 start = cmd->addr; 1530 end = start + size; 1531 if (end < start) 1532 return -EINVAL; 1533 1534 /* Since the mm is for the mirrored process, get a reference first. */ 1535 if (!mmget_not_zero(mm)) 1536 return -EINVAL; 1537 1538 /* 1539 * Register a temporary notifier to detect invalidations even if it 1540 * overlaps with other mmu_interval_notifiers. 1541 */ 1542 uptr = u64_to_user_ptr(cmd->ptr); 1543 for (addr = start; addr < end; addr = next) { 1544 unsigned long n; 1545 1546 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1547 range.start = addr; 1548 range.end = next; 1549 1550 ret = dmirror_range_snapshot(dmirror, &range, perm); 1551 if (ret) 1552 break; 1553 1554 n = (range.end - range.start) >> PAGE_SHIFT; 1555 if (copy_to_user(uptr, perm, n)) { 1556 ret = -EFAULT; 1557 break; 1558 } 1559 1560 cmd->cpages += n; 1561 uptr += n; 1562 } 1563 mmput(mm); 1564 1565 return ret; 1566 } 1567 1568 /* Removes free pages from the free list so they can't be re-allocated */ 1569 static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) 1570 { 1571 struct dmirror_device *mdevice = devmem->mdevice; 1572 struct page *page; 1573 struct folio *folio; 1574 1575 1576 for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio)) 1577 if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem) 1578 mdevice->free_folios = folio_zone_device_data(folio); 1579 for (page = mdevice->free_pages; page; page = page->zone_device_data) 1580 if (dmirror_page_to_chunk(page) == devmem) 1581 mdevice->free_pages = page->zone_device_data; 1582 } 1583 1584 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) 1585 { 1586 unsigned int i; 1587 1588 mutex_lock(&mdevice->devmem_lock); 1589 if (mdevice->devmem_chunks) { 1590 for (i = 0; i < mdevice->devmem_count; i++) { 1591 struct dmirror_chunk *devmem = 1592 mdevice->devmem_chunks[i]; 1593 1594 spin_lock(&mdevice->lock); 1595 devmem->remove = true; 1596 dmirror_remove_free_pages(devmem); 1597 spin_unlock(&mdevice->lock); 1598 1599 dmirror_device_evict_chunk(devmem); 1600 memunmap_pages(&devmem->pagemap); 1601 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1602 release_mem_region(devmem->pagemap.range.start, 1603 range_len(&devmem->pagemap.range)); 1604 kfree(devmem); 1605 } 1606 mdevice->devmem_count = 0; 1607 mdevice->devmem_capacity = 0; 1608 mdevice->free_pages = NULL; 1609 mdevice->free_folios = NULL; 1610 kfree(mdevice->devmem_chunks); 1611 mdevice->devmem_chunks = NULL; 1612 } 1613 mutex_unlock(&mdevice->devmem_lock); 1614 } 1615 1616 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1617 unsigned int command, 1618 unsigned long arg) 1619 { 1620 void __user *uarg = (void __user *)arg; 1621 struct hmm_dmirror_cmd cmd; 1622 struct dmirror *dmirror; 1623 int ret; 1624 1625 dmirror = filp->private_data; 1626 if (!dmirror) 1627 return -EINVAL; 1628 1629 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1630 return -EFAULT; 1631 1632 if (cmd.addr & ~PAGE_MASK) 1633 return -EINVAL; 1634 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1635 return -EINVAL; 1636 1637 cmd.cpages = 0; 1638 cmd.faults = 0; 1639 1640 switch (command) { 1641 case HMM_DMIRROR_READ: 1642 ret = dmirror_read(dmirror, &cmd); 1643 break; 1644 1645 case HMM_DMIRROR_WRITE: 1646 ret = dmirror_write(dmirror, &cmd); 1647 break; 1648 1649 case HMM_DMIRROR_MIGRATE_TO_DEV: 1650 ret = dmirror_migrate_to_device(dmirror, &cmd); 1651 break; 1652 1653 case HMM_DMIRROR_MIGRATE_TO_SYS: 1654 ret = dmirror_migrate_to_system(dmirror, &cmd); 1655 break; 1656 1657 case HMM_DMIRROR_EXCLUSIVE: 1658 ret = dmirror_exclusive(dmirror, &cmd); 1659 break; 1660 1661 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1662 ret = dmirror_check_atomic(dmirror, cmd.addr, 1663 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1664 break; 1665 1666 case HMM_DMIRROR_SNAPSHOT: 1667 ret = dmirror_snapshot(dmirror, &cmd); 1668 break; 1669 1670 case HMM_DMIRROR_RELEASE: 1671 dmirror_device_remove_chunks(dmirror->mdevice); 1672 ret = 0; 1673 break; 1674 case HMM_DMIRROR_FLAGS: 1675 dmirror->flags = cmd.npages; 1676 ret = 0; 1677 break; 1678 case HMM_DMIRROR_READ_UNLOCKED: 1679 ret = dmirror_read_unlocked(dmirror, &cmd, 0); 1680 break; 1681 default: 1682 return -EINVAL; 1683 } 1684 if (ret) 1685 return ret; 1686 1687 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1688 return -EFAULT; 1689 1690 return 0; 1691 } 1692 1693 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1694 { 1695 unsigned long addr; 1696 1697 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1698 struct page *page; 1699 int ret; 1700 1701 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1702 if (!page) 1703 return -ENOMEM; 1704 1705 ret = vm_insert_page(vma, addr, page); 1706 if (ret) { 1707 __free_page(page); 1708 return ret; 1709 } 1710 put_page(page); 1711 } 1712 1713 return 0; 1714 } 1715 1716 static const struct file_operations dmirror_fops = { 1717 .open = dmirror_fops_open, 1718 .release = dmirror_fops_release, 1719 .mmap = dmirror_fops_mmap, 1720 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1721 .llseek = default_llseek, 1722 .owner = THIS_MODULE, 1723 }; 1724 1725 static void dmirror_devmem_free(struct folio *folio) 1726 { 1727 struct page *page = &folio->page; 1728 struct page *rpage = BACKING_PAGE(page); 1729 struct dmirror_device *mdevice; 1730 struct folio *rfolio = page_folio(rpage); 1731 unsigned int order = folio_order(rfolio); 1732 1733 if (rpage != page) { 1734 if (order) 1735 __free_pages(rpage, order); 1736 else 1737 __free_page(rpage); 1738 rpage = NULL; 1739 } 1740 1741 mdevice = dmirror_page_to_device(page); 1742 spin_lock(&mdevice->lock); 1743 1744 /* Return page to our allocator if not freeing the chunk */ 1745 if (!dmirror_page_to_chunk(page)->remove) { 1746 mdevice->cfree += 1 << order; 1747 if (order) { 1748 page->zone_device_data = mdevice->free_folios; 1749 mdevice->free_folios = page_folio(page); 1750 } else { 1751 page->zone_device_data = mdevice->free_pages; 1752 mdevice->free_pages = page; 1753 } 1754 } 1755 spin_unlock(&mdevice->lock); 1756 } 1757 1758 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1759 { 1760 struct migrate_vma args = { 0 }; 1761 struct page *rpage; 1762 struct dmirror *dmirror; 1763 vm_fault_t ret = 0; 1764 unsigned int order, nr; 1765 1766 /* 1767 * Normally, a device would use the page->zone_device_data to point to 1768 * the mirror but here we use it to hold the page for the simulated 1769 * device memory and that page holds the pointer to the mirror. 1770 */ 1771 rpage = folio_zone_device_data(page_folio(vmf->page)); 1772 dmirror = rpage->zone_device_data; 1773 1774 /* FIXME demonstrate how we can adjust migrate range */ 1775 order = folio_order(page_folio(vmf->page)); 1776 nr = 1 << order; 1777 1778 /* 1779 * When folios are partially mapped, we can't rely on the folio 1780 * order of vmf->page as the folio might not be fully split yet 1781 */ 1782 if (vmf->pte) { 1783 order = 0; 1784 nr = 1; 1785 } 1786 1787 /* 1788 * Consider a per-cpu cache of src and dst pfns, but with 1789 * large number of cpus that might not scale well. 1790 */ 1791 args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order)); 1792 args.vma = vmf->vma; 1793 args.end = args.start + (PAGE_SIZE << order); 1794 1795 nr = (args.end - args.start) >> PAGE_SHIFT; 1796 args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); 1797 args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL); 1798 args.pgmap_owner = dmirror->mdevice; 1799 args.flags = dmirror_select_device(dmirror); 1800 args.fault_page = vmf->page; 1801 1802 if (!args.src || !args.dst) { 1803 ret = VM_FAULT_OOM; 1804 goto err; 1805 } 1806 1807 if (order) 1808 args.flags |= MIGRATE_VMA_SELECT_COMPOUND; 1809 1810 /* 1811 * In practice migrate_vma_setup() should never fail unless the 1812 * test is wrong as it just tests some static VMA properties. 1813 */ 1814 if (migrate_vma_setup(&args)) { 1815 ret = VM_FAULT_SIGBUS; 1816 goto err; 1817 } 1818 1819 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1820 if (ret) { 1821 migrate_vma_finalize(&args); 1822 goto err; 1823 } 1824 migrate_vma_pages(&args); 1825 /* 1826 * No device finalize step is needed since 1827 * dmirror_devmem_fault_alloc_and_copy() will have already 1828 * invalidated the device page table. 1829 */ 1830 migrate_vma_finalize(&args); 1831 err: 1832 kfree(args.src); 1833 kfree(args.dst); 1834 return ret; 1835 } 1836 1837 static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail) 1838 { 1839 struct page *rpage = BACKING_PAGE(folio_page(head, 0)); 1840 struct page *rpage_tail; 1841 struct folio *rfolio; 1842 unsigned long offset = 0; 1843 1844 if (!rpage) { 1845 tail->page.zone_device_data = NULL; 1846 return; 1847 } 1848 1849 rfolio = page_folio(rpage); 1850 1851 if (tail == NULL) { 1852 folio_reset_order(rfolio); 1853 rfolio->mapping = NULL; 1854 folio_set_count(rfolio, 1); 1855 return; 1856 } 1857 1858 offset = folio_pfn(tail) - folio_pfn(head); 1859 1860 rpage_tail = folio_page(rfolio, offset); 1861 tail->page.zone_device_data = rpage_tail; 1862 rpage_tail->zone_device_data = rpage->zone_device_data; 1863 clear_compound_head(rpage_tail); 1864 rpage_tail->mapping = NULL; 1865 1866 folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping; 1867 tail->pgmap = head->pgmap; 1868 folio_set_count(page_folio(rpage_tail), 1); 1869 } 1870 1871 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1872 .folio_free = dmirror_devmem_free, 1873 .migrate_to_ram = dmirror_devmem_fault, 1874 .folio_split = dmirror_devmem_folio_split, 1875 }; 1876 1877 static void dmirror_device_release(struct device *dev) 1878 { 1879 struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device); 1880 1881 dmirror_device_remove_chunks(mdevice); 1882 } 1883 1884 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1885 { 1886 dev_t dev; 1887 int ret; 1888 1889 dev = MKDEV(MAJOR(dmirror_dev), id); 1890 mutex_init(&mdevice->devmem_lock); 1891 spin_lock_init(&mdevice->lock); 1892 1893 cdev_init(&mdevice->cdevice, &dmirror_fops); 1894 mdevice->cdevice.owner = THIS_MODULE; 1895 mdevice->device.release = dmirror_device_release; 1896 1897 device_initialize(&mdevice->device); 1898 mdevice->device.devt = dev; 1899 1900 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); 1901 if (ret) 1902 goto put_device; 1903 1904 /* Build a list of free ZONE_DEVICE struct pages */ 1905 ret = dmirror_allocate_chunk(mdevice, NULL, false); 1906 if (ret) 1907 goto put_device; 1908 1909 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); 1910 if (ret) 1911 goto put_device; 1912 1913 return 0; 1914 1915 put_device: 1916 put_device(&mdevice->device); 1917 return ret; 1918 } 1919 1920 static void dmirror_device_remove(struct dmirror_device *mdevice) 1921 { 1922 cdev_device_del(&mdevice->cdevice, &mdevice->device); 1923 put_device(&mdevice->device); 1924 } 1925 1926 static int __init hmm_dmirror_init(void) 1927 { 1928 int ret; 1929 int id = 0; 1930 int ndevices = 0; 1931 1932 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1933 "HMM_DMIRROR"); 1934 if (ret) 1935 goto err_unreg; 1936 1937 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1938 dmirror_devices[ndevices++].zone_device_type = 1939 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1940 dmirror_devices[ndevices++].zone_device_type = 1941 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1942 if (spm_addr_dev0 && spm_addr_dev1) { 1943 dmirror_devices[ndevices++].zone_device_type = 1944 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1945 dmirror_devices[ndevices++].zone_device_type = 1946 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1947 } 1948 for (id = 0; id < ndevices; id++) { 1949 ret = dmirror_device_init(dmirror_devices + id, id); 1950 if (ret) 1951 goto err_chrdev; 1952 } 1953 1954 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1955 return 0; 1956 1957 err_chrdev: 1958 while (--id >= 0) 1959 dmirror_device_remove(dmirror_devices + id); 1960 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1961 err_unreg: 1962 return ret; 1963 } 1964 1965 static void __exit hmm_dmirror_exit(void) 1966 { 1967 int id; 1968 1969 for (id = 0; id < DMIRROR_NDEVICES; id++) 1970 if (dmirror_devices[id].zone_device_type) 1971 dmirror_device_remove(dmirror_devices + id); 1972 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1973 } 1974 1975 module_init(hmm_dmirror_init); 1976 module_exit(hmm_dmirror_exit); 1977 MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module"); 1978 MODULE_LICENSE("GPL"); 1979