1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 }; 96 97 /* 98 * ZONE_DEVICE pages for migration and simulating device memory. 99 */ 100 struct dmirror_chunk { 101 struct dev_pagemap pagemap; 102 struct dmirror_device *mdevice; 103 bool remove; 104 }; 105 106 /* 107 * Per device data. 108 */ 109 struct dmirror_device { 110 struct cdev cdevice; 111 unsigned int zone_device_type; 112 struct device device; 113 114 unsigned int devmem_capacity; 115 unsigned int devmem_count; 116 struct dmirror_chunk **devmem_chunks; 117 struct mutex devmem_lock; /* protects the above */ 118 119 unsigned long calloc; 120 unsigned long cfree; 121 struct page *free_pages; 122 spinlock_t lock; /* protects the above */ 123 }; 124 125 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 126 127 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 128 unsigned long addr, 129 unsigned long size) 130 { 131 bounce->addr = addr; 132 bounce->size = size; 133 bounce->cpages = 0; 134 bounce->ptr = vmalloc(size); 135 if (!bounce->ptr) 136 return -ENOMEM; 137 return 0; 138 } 139 140 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 141 { 142 return (mdevice->zone_device_type == 143 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? true : false; 144 } 145 146 static enum migrate_vma_direction 147 dmirror_select_device(struct dmirror *dmirror) 148 { 149 return (dmirror->mdevice->zone_device_type == 150 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 151 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 152 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 153 } 154 155 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 156 { 157 vfree(bounce->ptr); 158 } 159 160 static int dmirror_fops_open(struct inode *inode, struct file *filp) 161 { 162 struct cdev *cdev = inode->i_cdev; 163 struct dmirror *dmirror; 164 int ret; 165 166 /* Mirror this process address space */ 167 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); 168 if (dmirror == NULL) 169 return -ENOMEM; 170 171 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 172 mutex_init(&dmirror->mutex); 173 xa_init(&dmirror->pt); 174 175 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 176 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 177 if (ret) { 178 kfree(dmirror); 179 return ret; 180 } 181 182 filp->private_data = dmirror; 183 return 0; 184 } 185 186 static int dmirror_fops_release(struct inode *inode, struct file *filp) 187 { 188 struct dmirror *dmirror = filp->private_data; 189 190 mmu_interval_notifier_remove(&dmirror->notifier); 191 xa_destroy(&dmirror->pt); 192 kfree(dmirror); 193 return 0; 194 } 195 196 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) 197 { 198 return container_of(page->pgmap, struct dmirror_chunk, pagemap); 199 } 200 201 static struct dmirror_device *dmirror_page_to_device(struct page *page) 202 203 { 204 return dmirror_page_to_chunk(page)->mdevice; 205 } 206 207 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 208 { 209 unsigned long *pfns = range->hmm_pfns; 210 unsigned long pfn; 211 212 for (pfn = (range->start >> PAGE_SHIFT); 213 pfn < (range->end >> PAGE_SHIFT); 214 pfn++, pfns++) { 215 struct page *page; 216 void *entry; 217 218 /* 219 * Since we asked for hmm_range_fault() to populate pages, 220 * it shouldn't return an error entry on success. 221 */ 222 WARN_ON(*pfns & HMM_PFN_ERROR); 223 WARN_ON(!(*pfns & HMM_PFN_VALID)); 224 225 page = hmm_pfn_to_page(*pfns); 226 WARN_ON(!page); 227 228 entry = page; 229 if (*pfns & HMM_PFN_WRITE) 230 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 231 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 232 return -EFAULT; 233 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 234 if (xa_is_err(entry)) 235 return xa_err(entry); 236 } 237 238 return 0; 239 } 240 241 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 242 unsigned long end) 243 { 244 unsigned long pfn; 245 void *entry; 246 247 /* 248 * The XArray doesn't hold references to pages since it relies on 249 * the mmu notifier to clear page pointers when they become stale. 250 * Therefore, it is OK to just clear the entry. 251 */ 252 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 253 end >> PAGE_SHIFT) 254 xa_erase(&dmirror->pt, pfn); 255 } 256 257 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 258 const struct mmu_notifier_range *range, 259 unsigned long cur_seq) 260 { 261 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 262 263 /* 264 * Ignore invalidation callbacks for device private pages since 265 * the invalidation is handled as part of the migration process. 266 */ 267 if (range->event == MMU_NOTIFY_MIGRATE && 268 range->owner == dmirror->mdevice) 269 return true; 270 271 if (mmu_notifier_range_blockable(range)) 272 mutex_lock(&dmirror->mutex); 273 else if (!mutex_trylock(&dmirror->mutex)) 274 return false; 275 276 mmu_interval_set_seq(mni, cur_seq); 277 dmirror_do_update(dmirror, range->start, range->end); 278 279 mutex_unlock(&dmirror->mutex); 280 return true; 281 } 282 283 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 284 .invalidate = dmirror_interval_invalidate, 285 }; 286 287 static int dmirror_range_fault(struct dmirror *dmirror, 288 struct hmm_range *range) 289 { 290 struct mm_struct *mm = dmirror->notifier.mm; 291 unsigned long timeout = 292 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 293 int ret; 294 295 while (true) { 296 if (time_after(jiffies, timeout)) { 297 ret = -EBUSY; 298 goto out; 299 } 300 301 range->notifier_seq = mmu_interval_read_begin(range->notifier); 302 mmap_read_lock(mm); 303 ret = hmm_range_fault(range); 304 mmap_read_unlock(mm); 305 if (ret) { 306 if (ret == -EBUSY) 307 continue; 308 goto out; 309 } 310 311 mutex_lock(&dmirror->mutex); 312 if (mmu_interval_read_retry(range->notifier, 313 range->notifier_seq)) { 314 mutex_unlock(&dmirror->mutex); 315 continue; 316 } 317 break; 318 } 319 320 ret = dmirror_do_fault(dmirror, range); 321 322 mutex_unlock(&dmirror->mutex); 323 out: 324 return ret; 325 } 326 327 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 328 unsigned long end, bool write) 329 { 330 struct mm_struct *mm = dmirror->notifier.mm; 331 unsigned long addr; 332 unsigned long pfns[64]; 333 struct hmm_range range = { 334 .notifier = &dmirror->notifier, 335 .hmm_pfns = pfns, 336 .pfn_flags_mask = 0, 337 .default_flags = 338 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 339 .dev_private_owner = dmirror->mdevice, 340 }; 341 int ret = 0; 342 343 /* Since the mm is for the mirrored process, get a reference first. */ 344 if (!mmget_not_zero(mm)) 345 return 0; 346 347 for (addr = start; addr < end; addr = range.end) { 348 range.start = addr; 349 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 350 351 ret = dmirror_range_fault(dmirror, &range); 352 if (ret) 353 break; 354 } 355 356 mmput(mm); 357 return ret; 358 } 359 360 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 361 unsigned long end, struct dmirror_bounce *bounce) 362 { 363 unsigned long pfn; 364 void *ptr; 365 366 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 367 368 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 369 void *entry; 370 struct page *page; 371 372 entry = xa_load(&dmirror->pt, pfn); 373 page = xa_untag_pointer(entry); 374 if (!page) 375 return -ENOENT; 376 377 memcpy_from_page(ptr, page, 0, PAGE_SIZE); 378 379 ptr += PAGE_SIZE; 380 bounce->cpages++; 381 } 382 383 return 0; 384 } 385 386 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 387 { 388 struct dmirror_bounce bounce; 389 unsigned long start, end; 390 unsigned long size = cmd->npages << PAGE_SHIFT; 391 int ret; 392 393 start = cmd->addr; 394 end = start + size; 395 if (end < start) 396 return -EINVAL; 397 398 ret = dmirror_bounce_init(&bounce, start, size); 399 if (ret) 400 return ret; 401 402 while (1) { 403 mutex_lock(&dmirror->mutex); 404 ret = dmirror_do_read(dmirror, start, end, &bounce); 405 mutex_unlock(&dmirror->mutex); 406 if (ret != -ENOENT) 407 break; 408 409 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 410 ret = dmirror_fault(dmirror, start, end, false); 411 if (ret) 412 break; 413 cmd->faults++; 414 } 415 416 if (ret == 0) { 417 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 418 bounce.size)) 419 ret = -EFAULT; 420 } 421 cmd->cpages = bounce.cpages; 422 dmirror_bounce_fini(&bounce); 423 return ret; 424 } 425 426 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 427 unsigned long end, struct dmirror_bounce *bounce) 428 { 429 unsigned long pfn; 430 void *ptr; 431 432 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 433 434 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 435 void *entry; 436 struct page *page; 437 438 entry = xa_load(&dmirror->pt, pfn); 439 page = xa_untag_pointer(entry); 440 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 441 return -ENOENT; 442 443 memcpy_to_page(page, 0, ptr, PAGE_SIZE); 444 445 ptr += PAGE_SIZE; 446 bounce->cpages++; 447 } 448 449 return 0; 450 } 451 452 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 453 { 454 struct dmirror_bounce bounce; 455 unsigned long start, end; 456 unsigned long size = cmd->npages << PAGE_SHIFT; 457 int ret; 458 459 start = cmd->addr; 460 end = start + size; 461 if (end < start) 462 return -EINVAL; 463 464 ret = dmirror_bounce_init(&bounce, start, size); 465 if (ret) 466 return ret; 467 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 468 bounce.size)) { 469 ret = -EFAULT; 470 goto fini; 471 } 472 473 while (1) { 474 mutex_lock(&dmirror->mutex); 475 ret = dmirror_do_write(dmirror, start, end, &bounce); 476 mutex_unlock(&dmirror->mutex); 477 if (ret != -ENOENT) 478 break; 479 480 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 481 ret = dmirror_fault(dmirror, start, end, true); 482 if (ret) 483 break; 484 cmd->faults++; 485 } 486 487 fini: 488 cmd->cpages = bounce.cpages; 489 dmirror_bounce_fini(&bounce); 490 return ret; 491 } 492 493 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 494 struct page **ppage) 495 { 496 struct dmirror_chunk *devmem; 497 struct resource *res = NULL; 498 unsigned long pfn; 499 unsigned long pfn_first; 500 unsigned long pfn_last; 501 void *ptr; 502 int ret = -ENOMEM; 503 504 devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); 505 if (!devmem) 506 return ret; 507 508 switch (mdevice->zone_device_type) { 509 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 510 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 511 "hmm_dmirror"); 512 if (IS_ERR_OR_NULL(res)) 513 goto err_devmem; 514 devmem->pagemap.range.start = res->start; 515 devmem->pagemap.range.end = res->end; 516 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 517 break; 518 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 519 devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? 520 spm_addr_dev0 : 521 spm_addr_dev1; 522 devmem->pagemap.range.end = devmem->pagemap.range.start + 523 DEVMEM_CHUNK_SIZE - 1; 524 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 525 break; 526 default: 527 ret = -EINVAL; 528 goto err_devmem; 529 } 530 531 devmem->pagemap.nr_range = 1; 532 devmem->pagemap.ops = &dmirror_devmem_ops; 533 devmem->pagemap.owner = mdevice; 534 535 mutex_lock(&mdevice->devmem_lock); 536 537 if (mdevice->devmem_count == mdevice->devmem_capacity) { 538 struct dmirror_chunk **new_chunks; 539 unsigned int new_capacity; 540 541 new_capacity = mdevice->devmem_capacity + 542 DEVMEM_CHUNKS_RESERVE; 543 new_chunks = krealloc(mdevice->devmem_chunks, 544 sizeof(new_chunks[0]) * new_capacity, 545 GFP_KERNEL); 546 if (!new_chunks) 547 goto err_release; 548 mdevice->devmem_capacity = new_capacity; 549 mdevice->devmem_chunks = new_chunks; 550 } 551 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 552 if (IS_ERR_OR_NULL(ptr)) { 553 if (ptr) 554 ret = PTR_ERR(ptr); 555 else 556 ret = -EFAULT; 557 goto err_release; 558 } 559 560 devmem->mdevice = mdevice; 561 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 562 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 563 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 564 565 mutex_unlock(&mdevice->devmem_lock); 566 567 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 568 DEVMEM_CHUNK_SIZE / (1024 * 1024), 569 mdevice->devmem_count, 570 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 571 pfn_first, pfn_last); 572 573 spin_lock(&mdevice->lock); 574 for (pfn = pfn_first; pfn < pfn_last; pfn++) { 575 struct page *page = pfn_to_page(pfn); 576 577 page->zone_device_data = mdevice->free_pages; 578 mdevice->free_pages = page; 579 } 580 if (ppage) { 581 *ppage = mdevice->free_pages; 582 mdevice->free_pages = (*ppage)->zone_device_data; 583 mdevice->calloc++; 584 } 585 spin_unlock(&mdevice->lock); 586 587 return 0; 588 589 err_release: 590 mutex_unlock(&mdevice->devmem_lock); 591 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 592 release_mem_region(devmem->pagemap.range.start, 593 range_len(&devmem->pagemap.range)); 594 err_devmem: 595 kfree(devmem); 596 597 return ret; 598 } 599 600 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) 601 { 602 struct page *dpage = NULL; 603 struct page *rpage = NULL; 604 605 /* 606 * For ZONE_DEVICE private type, this is a fake device so we allocate 607 * real system memory to store our device memory. 608 * For ZONE_DEVICE coherent type we use the actual dpage to store the 609 * data and ignore rpage. 610 */ 611 if (dmirror_is_private_zone(mdevice)) { 612 rpage = alloc_page(GFP_HIGHUSER); 613 if (!rpage) 614 return NULL; 615 } 616 spin_lock(&mdevice->lock); 617 618 if (mdevice->free_pages) { 619 dpage = mdevice->free_pages; 620 mdevice->free_pages = dpage->zone_device_data; 621 mdevice->calloc++; 622 spin_unlock(&mdevice->lock); 623 } else { 624 spin_unlock(&mdevice->lock); 625 if (dmirror_allocate_chunk(mdevice, &dpage)) 626 goto error; 627 } 628 629 zone_device_page_init(dpage); 630 dpage->zone_device_data = rpage; 631 return dpage; 632 633 error: 634 if (rpage) 635 __free_page(rpage); 636 return NULL; 637 } 638 639 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 640 struct dmirror *dmirror) 641 { 642 struct dmirror_device *mdevice = dmirror->mdevice; 643 const unsigned long *src = args->src; 644 unsigned long *dst = args->dst; 645 unsigned long addr; 646 647 for (addr = args->start; addr < args->end; addr += PAGE_SIZE, 648 src++, dst++) { 649 struct page *spage; 650 struct page *dpage; 651 struct page *rpage; 652 653 if (!(*src & MIGRATE_PFN_MIGRATE)) 654 continue; 655 656 /* 657 * Note that spage might be NULL which is OK since it is an 658 * unallocated pte_none() or read-only zero page. 659 */ 660 spage = migrate_pfn_to_page(*src); 661 if (WARN(spage && is_zone_device_page(spage), 662 "page already in device spage pfn: 0x%lx\n", 663 page_to_pfn(spage))) 664 continue; 665 666 dpage = dmirror_devmem_alloc_page(mdevice); 667 if (!dpage) 668 continue; 669 670 rpage = BACKING_PAGE(dpage); 671 if (spage) 672 copy_highpage(rpage, spage); 673 else 674 clear_highpage(rpage); 675 676 /* 677 * Normally, a device would use the page->zone_device_data to 678 * point to the mirror but here we use it to hold the page for 679 * the simulated device memory and that page holds the pointer 680 * to the mirror. 681 */ 682 rpage->zone_device_data = dmirror; 683 684 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 685 page_to_pfn(spage), page_to_pfn(dpage)); 686 *dst = migrate_pfn(page_to_pfn(dpage)); 687 if ((*src & MIGRATE_PFN_WRITE) || 688 (!spage && args->vma->vm_flags & VM_WRITE)) 689 *dst |= MIGRATE_PFN_WRITE; 690 } 691 } 692 693 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 694 unsigned long end) 695 { 696 unsigned long pfn; 697 698 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 699 void *entry; 700 701 entry = xa_load(&dmirror->pt, pfn); 702 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 703 return -EPERM; 704 } 705 706 return 0; 707 } 708 709 static int dmirror_atomic_map(unsigned long start, unsigned long end, 710 struct page **pages, struct dmirror *dmirror) 711 { 712 unsigned long pfn, mapped = 0; 713 int i; 714 715 /* Map the migrated pages into the device's page tables. */ 716 mutex_lock(&dmirror->mutex); 717 718 for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) { 719 void *entry; 720 721 if (!pages[i]) 722 continue; 723 724 entry = pages[i]; 725 entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); 726 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 727 if (xa_is_err(entry)) { 728 mutex_unlock(&dmirror->mutex); 729 return xa_err(entry); 730 } 731 732 mapped++; 733 } 734 735 mutex_unlock(&dmirror->mutex); 736 return mapped; 737 } 738 739 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 740 struct dmirror *dmirror) 741 { 742 unsigned long start = args->start; 743 unsigned long end = args->end; 744 const unsigned long *src = args->src; 745 const unsigned long *dst = args->dst; 746 unsigned long pfn; 747 748 /* Map the migrated pages into the device's page tables. */ 749 mutex_lock(&dmirror->mutex); 750 751 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, 752 src++, dst++) { 753 struct page *dpage; 754 void *entry; 755 756 if (!(*src & MIGRATE_PFN_MIGRATE)) 757 continue; 758 759 dpage = migrate_pfn_to_page(*dst); 760 if (!dpage) 761 continue; 762 763 entry = BACKING_PAGE(dpage); 764 if (*dst & MIGRATE_PFN_WRITE) 765 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 766 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 767 if (xa_is_err(entry)) { 768 mutex_unlock(&dmirror->mutex); 769 return xa_err(entry); 770 } 771 } 772 773 mutex_unlock(&dmirror->mutex); 774 return 0; 775 } 776 777 static int dmirror_exclusive(struct dmirror *dmirror, 778 struct hmm_dmirror_cmd *cmd) 779 { 780 unsigned long start, end, addr; 781 unsigned long size = cmd->npages << PAGE_SHIFT; 782 struct mm_struct *mm = dmirror->notifier.mm; 783 struct page *pages[64]; 784 struct dmirror_bounce bounce; 785 unsigned long next; 786 int ret; 787 788 start = cmd->addr; 789 end = start + size; 790 if (end < start) 791 return -EINVAL; 792 793 /* Since the mm is for the mirrored process, get a reference first. */ 794 if (!mmget_not_zero(mm)) 795 return -EINVAL; 796 797 mmap_read_lock(mm); 798 for (addr = start; addr < end; addr = next) { 799 unsigned long mapped = 0; 800 int i; 801 802 next = min(end, addr + (ARRAY_SIZE(pages) << PAGE_SHIFT)); 803 804 ret = make_device_exclusive_range(mm, addr, next, pages, NULL); 805 /* 806 * Do dmirror_atomic_map() iff all pages are marked for 807 * exclusive access to avoid accessing uninitialized 808 * fields of pages. 809 */ 810 if (ret == (next - addr) >> PAGE_SHIFT) 811 mapped = dmirror_atomic_map(addr, next, pages, dmirror); 812 for (i = 0; i < ret; i++) { 813 if (pages[i]) { 814 unlock_page(pages[i]); 815 put_page(pages[i]); 816 } 817 } 818 819 if (addr + (mapped << PAGE_SHIFT) < next) { 820 mmap_read_unlock(mm); 821 mmput(mm); 822 return -EBUSY; 823 } 824 } 825 mmap_read_unlock(mm); 826 mmput(mm); 827 828 /* Return the migrated data for verification. */ 829 ret = dmirror_bounce_init(&bounce, start, size); 830 if (ret) 831 return ret; 832 mutex_lock(&dmirror->mutex); 833 ret = dmirror_do_read(dmirror, start, end, &bounce); 834 mutex_unlock(&dmirror->mutex); 835 if (ret == 0) { 836 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 837 bounce.size)) 838 ret = -EFAULT; 839 } 840 841 cmd->cpages = bounce.cpages; 842 dmirror_bounce_fini(&bounce); 843 return ret; 844 } 845 846 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 847 struct dmirror *dmirror) 848 { 849 const unsigned long *src = args->src; 850 unsigned long *dst = args->dst; 851 unsigned long start = args->start; 852 unsigned long end = args->end; 853 unsigned long addr; 854 855 for (addr = start; addr < end; addr += PAGE_SIZE, 856 src++, dst++) { 857 struct page *dpage, *spage; 858 859 spage = migrate_pfn_to_page(*src); 860 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) 861 continue; 862 863 if (WARN_ON(!is_device_private_page(spage) && 864 !is_device_coherent_page(spage))) 865 continue; 866 spage = BACKING_PAGE(spage); 867 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 868 if (!dpage) 869 continue; 870 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", 871 page_to_pfn(spage), page_to_pfn(dpage)); 872 873 lock_page(dpage); 874 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 875 copy_highpage(dpage, spage); 876 *dst = migrate_pfn(page_to_pfn(dpage)); 877 if (*src & MIGRATE_PFN_WRITE) 878 *dst |= MIGRATE_PFN_WRITE; 879 } 880 return 0; 881 } 882 883 static unsigned long 884 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 885 { 886 unsigned long cpages = 0; 887 unsigned long i; 888 889 for (i = 0; i < migrate->npages; i++) { 890 if (migrate->src[i] & MIGRATE_PFN_VALID && 891 migrate->src[i] & MIGRATE_PFN_MIGRATE) 892 cpages++; 893 } 894 return cpages; 895 } 896 897 static int dmirror_migrate_to_system(struct dmirror *dmirror, 898 struct hmm_dmirror_cmd *cmd) 899 { 900 unsigned long start, end, addr; 901 unsigned long size = cmd->npages << PAGE_SHIFT; 902 struct mm_struct *mm = dmirror->notifier.mm; 903 struct vm_area_struct *vma; 904 unsigned long src_pfns[64] = { 0 }; 905 unsigned long dst_pfns[64] = { 0 }; 906 struct migrate_vma args = { 0 }; 907 unsigned long next; 908 int ret; 909 910 start = cmd->addr; 911 end = start + size; 912 if (end < start) 913 return -EINVAL; 914 915 /* Since the mm is for the mirrored process, get a reference first. */ 916 if (!mmget_not_zero(mm)) 917 return -EINVAL; 918 919 cmd->cpages = 0; 920 mmap_read_lock(mm); 921 for (addr = start; addr < end; addr = next) { 922 vma = vma_lookup(mm, addr); 923 if (!vma || !(vma->vm_flags & VM_READ)) { 924 ret = -EINVAL; 925 goto out; 926 } 927 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 928 if (next > vma->vm_end) 929 next = vma->vm_end; 930 931 args.vma = vma; 932 args.src = src_pfns; 933 args.dst = dst_pfns; 934 args.start = addr; 935 args.end = next; 936 args.pgmap_owner = dmirror->mdevice; 937 args.flags = dmirror_select_device(dmirror); 938 939 ret = migrate_vma_setup(&args); 940 if (ret) 941 goto out; 942 943 pr_debug("Migrating from device mem to sys mem\n"); 944 dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 945 946 migrate_vma_pages(&args); 947 cmd->cpages += dmirror_successful_migrated_pages(&args); 948 migrate_vma_finalize(&args); 949 } 950 out: 951 mmap_read_unlock(mm); 952 mmput(mm); 953 954 return ret; 955 } 956 957 static int dmirror_migrate_to_device(struct dmirror *dmirror, 958 struct hmm_dmirror_cmd *cmd) 959 { 960 unsigned long start, end, addr; 961 unsigned long size = cmd->npages << PAGE_SHIFT; 962 struct mm_struct *mm = dmirror->notifier.mm; 963 struct vm_area_struct *vma; 964 unsigned long src_pfns[64] = { 0 }; 965 unsigned long dst_pfns[64] = { 0 }; 966 struct dmirror_bounce bounce; 967 struct migrate_vma args = { 0 }; 968 unsigned long next; 969 int ret; 970 971 start = cmd->addr; 972 end = start + size; 973 if (end < start) 974 return -EINVAL; 975 976 /* Since the mm is for the mirrored process, get a reference first. */ 977 if (!mmget_not_zero(mm)) 978 return -EINVAL; 979 980 mmap_read_lock(mm); 981 for (addr = start; addr < end; addr = next) { 982 vma = vma_lookup(mm, addr); 983 if (!vma || !(vma->vm_flags & VM_READ)) { 984 ret = -EINVAL; 985 goto out; 986 } 987 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 988 if (next > vma->vm_end) 989 next = vma->vm_end; 990 991 args.vma = vma; 992 args.src = src_pfns; 993 args.dst = dst_pfns; 994 args.start = addr; 995 args.end = next; 996 args.pgmap_owner = dmirror->mdevice; 997 args.flags = MIGRATE_VMA_SELECT_SYSTEM; 998 ret = migrate_vma_setup(&args); 999 if (ret) 1000 goto out; 1001 1002 pr_debug("Migrating from sys mem to device mem\n"); 1003 dmirror_migrate_alloc_and_copy(&args, dmirror); 1004 migrate_vma_pages(&args); 1005 dmirror_migrate_finalize_and_map(&args, dmirror); 1006 migrate_vma_finalize(&args); 1007 } 1008 mmap_read_unlock(mm); 1009 mmput(mm); 1010 1011 /* 1012 * Return the migrated data for verification. 1013 * Only for pages in device zone 1014 */ 1015 ret = dmirror_bounce_init(&bounce, start, size); 1016 if (ret) 1017 return ret; 1018 mutex_lock(&dmirror->mutex); 1019 ret = dmirror_do_read(dmirror, start, end, &bounce); 1020 mutex_unlock(&dmirror->mutex); 1021 if (ret == 0) { 1022 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1023 bounce.size)) 1024 ret = -EFAULT; 1025 } 1026 cmd->cpages = bounce.cpages; 1027 dmirror_bounce_fini(&bounce); 1028 return ret; 1029 1030 out: 1031 mmap_read_unlock(mm); 1032 mmput(mm); 1033 return ret; 1034 } 1035 1036 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1037 unsigned char *perm, unsigned long entry) 1038 { 1039 struct page *page; 1040 1041 if (entry & HMM_PFN_ERROR) { 1042 *perm = HMM_DMIRROR_PROT_ERROR; 1043 return; 1044 } 1045 if (!(entry & HMM_PFN_VALID)) { 1046 *perm = HMM_DMIRROR_PROT_NONE; 1047 return; 1048 } 1049 1050 page = hmm_pfn_to_page(entry); 1051 if (is_device_private_page(page)) { 1052 /* Is the page migrated to this device or some other? */ 1053 if (dmirror->mdevice == dmirror_page_to_device(page)) 1054 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1055 else 1056 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1057 } else if (is_device_coherent_page(page)) { 1058 /* Is the page migrated to this device or some other? */ 1059 if (dmirror->mdevice == dmirror_page_to_device(page)) 1060 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1061 else 1062 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1063 } else if (is_zero_pfn(page_to_pfn(page))) 1064 *perm = HMM_DMIRROR_PROT_ZERO; 1065 else 1066 *perm = HMM_DMIRROR_PROT_NONE; 1067 if (entry & HMM_PFN_WRITE) 1068 *perm |= HMM_DMIRROR_PROT_WRITE; 1069 else 1070 *perm |= HMM_DMIRROR_PROT_READ; 1071 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1072 *perm |= HMM_DMIRROR_PROT_PMD; 1073 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1074 *perm |= HMM_DMIRROR_PROT_PUD; 1075 } 1076 1077 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1078 const struct mmu_notifier_range *range, 1079 unsigned long cur_seq) 1080 { 1081 struct dmirror_interval *dmi = 1082 container_of(mni, struct dmirror_interval, notifier); 1083 struct dmirror *dmirror = dmi->dmirror; 1084 1085 if (mmu_notifier_range_blockable(range)) 1086 mutex_lock(&dmirror->mutex); 1087 else if (!mutex_trylock(&dmirror->mutex)) 1088 return false; 1089 1090 /* 1091 * Snapshots only need to set the sequence number since any 1092 * invalidation in the interval invalidates the whole snapshot. 1093 */ 1094 mmu_interval_set_seq(mni, cur_seq); 1095 1096 mutex_unlock(&dmirror->mutex); 1097 return true; 1098 } 1099 1100 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1101 .invalidate = dmirror_snapshot_invalidate, 1102 }; 1103 1104 static int dmirror_range_snapshot(struct dmirror *dmirror, 1105 struct hmm_range *range, 1106 unsigned char *perm) 1107 { 1108 struct mm_struct *mm = dmirror->notifier.mm; 1109 struct dmirror_interval notifier; 1110 unsigned long timeout = 1111 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1112 unsigned long i; 1113 unsigned long n; 1114 int ret = 0; 1115 1116 notifier.dmirror = dmirror; 1117 range->notifier = ¬ifier.notifier; 1118 1119 ret = mmu_interval_notifier_insert(range->notifier, mm, 1120 range->start, range->end - range->start, 1121 &dmirror_mrn_ops); 1122 if (ret) 1123 return ret; 1124 1125 while (true) { 1126 if (time_after(jiffies, timeout)) { 1127 ret = -EBUSY; 1128 goto out; 1129 } 1130 1131 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1132 1133 mmap_read_lock(mm); 1134 ret = hmm_range_fault(range); 1135 mmap_read_unlock(mm); 1136 if (ret) { 1137 if (ret == -EBUSY) 1138 continue; 1139 goto out; 1140 } 1141 1142 mutex_lock(&dmirror->mutex); 1143 if (mmu_interval_read_retry(range->notifier, 1144 range->notifier_seq)) { 1145 mutex_unlock(&dmirror->mutex); 1146 continue; 1147 } 1148 break; 1149 } 1150 1151 n = (range->end - range->start) >> PAGE_SHIFT; 1152 for (i = 0; i < n; i++) 1153 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1154 1155 mutex_unlock(&dmirror->mutex); 1156 out: 1157 mmu_interval_notifier_remove(range->notifier); 1158 return ret; 1159 } 1160 1161 static int dmirror_snapshot(struct dmirror *dmirror, 1162 struct hmm_dmirror_cmd *cmd) 1163 { 1164 struct mm_struct *mm = dmirror->notifier.mm; 1165 unsigned long start, end; 1166 unsigned long size = cmd->npages << PAGE_SHIFT; 1167 unsigned long addr; 1168 unsigned long next; 1169 unsigned long pfns[64]; 1170 unsigned char perm[64]; 1171 char __user *uptr; 1172 struct hmm_range range = { 1173 .hmm_pfns = pfns, 1174 .dev_private_owner = dmirror->mdevice, 1175 }; 1176 int ret = 0; 1177 1178 start = cmd->addr; 1179 end = start + size; 1180 if (end < start) 1181 return -EINVAL; 1182 1183 /* Since the mm is for the mirrored process, get a reference first. */ 1184 if (!mmget_not_zero(mm)) 1185 return -EINVAL; 1186 1187 /* 1188 * Register a temporary notifier to detect invalidations even if it 1189 * overlaps with other mmu_interval_notifiers. 1190 */ 1191 uptr = u64_to_user_ptr(cmd->ptr); 1192 for (addr = start; addr < end; addr = next) { 1193 unsigned long n; 1194 1195 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1196 range.start = addr; 1197 range.end = next; 1198 1199 ret = dmirror_range_snapshot(dmirror, &range, perm); 1200 if (ret) 1201 break; 1202 1203 n = (range.end - range.start) >> PAGE_SHIFT; 1204 if (copy_to_user(uptr, perm, n)) { 1205 ret = -EFAULT; 1206 break; 1207 } 1208 1209 cmd->cpages += n; 1210 uptr += n; 1211 } 1212 mmput(mm); 1213 1214 return ret; 1215 } 1216 1217 static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) 1218 { 1219 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; 1220 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; 1221 unsigned long npages = end_pfn - start_pfn + 1; 1222 unsigned long i; 1223 unsigned long *src_pfns; 1224 unsigned long *dst_pfns; 1225 1226 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); 1227 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); 1228 1229 migrate_device_range(src_pfns, start_pfn, npages); 1230 for (i = 0; i < npages; i++) { 1231 struct page *dpage, *spage; 1232 1233 spage = migrate_pfn_to_page(src_pfns[i]); 1234 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 1235 continue; 1236 1237 if (WARN_ON(!is_device_private_page(spage) && 1238 !is_device_coherent_page(spage))) 1239 continue; 1240 spage = BACKING_PAGE(spage); 1241 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); 1242 lock_page(dpage); 1243 copy_highpage(dpage, spage); 1244 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); 1245 if (src_pfns[i] & MIGRATE_PFN_WRITE) 1246 dst_pfns[i] |= MIGRATE_PFN_WRITE; 1247 } 1248 migrate_device_pages(src_pfns, dst_pfns, npages); 1249 migrate_device_finalize(src_pfns, dst_pfns, npages); 1250 kvfree(src_pfns); 1251 kvfree(dst_pfns); 1252 } 1253 1254 /* Removes free pages from the free list so they can't be re-allocated */ 1255 static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) 1256 { 1257 struct dmirror_device *mdevice = devmem->mdevice; 1258 struct page *page; 1259 1260 for (page = mdevice->free_pages; page; page = page->zone_device_data) 1261 if (dmirror_page_to_chunk(page) == devmem) 1262 mdevice->free_pages = page->zone_device_data; 1263 } 1264 1265 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) 1266 { 1267 unsigned int i; 1268 1269 mutex_lock(&mdevice->devmem_lock); 1270 if (mdevice->devmem_chunks) { 1271 for (i = 0; i < mdevice->devmem_count; i++) { 1272 struct dmirror_chunk *devmem = 1273 mdevice->devmem_chunks[i]; 1274 1275 spin_lock(&mdevice->lock); 1276 devmem->remove = true; 1277 dmirror_remove_free_pages(devmem); 1278 spin_unlock(&mdevice->lock); 1279 1280 dmirror_device_evict_chunk(devmem); 1281 memunmap_pages(&devmem->pagemap); 1282 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1283 release_mem_region(devmem->pagemap.range.start, 1284 range_len(&devmem->pagemap.range)); 1285 kfree(devmem); 1286 } 1287 mdevice->devmem_count = 0; 1288 mdevice->devmem_capacity = 0; 1289 mdevice->free_pages = NULL; 1290 kfree(mdevice->devmem_chunks); 1291 mdevice->devmem_chunks = NULL; 1292 } 1293 mutex_unlock(&mdevice->devmem_lock); 1294 } 1295 1296 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1297 unsigned int command, 1298 unsigned long arg) 1299 { 1300 void __user *uarg = (void __user *)arg; 1301 struct hmm_dmirror_cmd cmd; 1302 struct dmirror *dmirror; 1303 int ret; 1304 1305 dmirror = filp->private_data; 1306 if (!dmirror) 1307 return -EINVAL; 1308 1309 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1310 return -EFAULT; 1311 1312 if (cmd.addr & ~PAGE_MASK) 1313 return -EINVAL; 1314 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1315 return -EINVAL; 1316 1317 cmd.cpages = 0; 1318 cmd.faults = 0; 1319 1320 switch (command) { 1321 case HMM_DMIRROR_READ: 1322 ret = dmirror_read(dmirror, &cmd); 1323 break; 1324 1325 case HMM_DMIRROR_WRITE: 1326 ret = dmirror_write(dmirror, &cmd); 1327 break; 1328 1329 case HMM_DMIRROR_MIGRATE_TO_DEV: 1330 ret = dmirror_migrate_to_device(dmirror, &cmd); 1331 break; 1332 1333 case HMM_DMIRROR_MIGRATE_TO_SYS: 1334 ret = dmirror_migrate_to_system(dmirror, &cmd); 1335 break; 1336 1337 case HMM_DMIRROR_EXCLUSIVE: 1338 ret = dmirror_exclusive(dmirror, &cmd); 1339 break; 1340 1341 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1342 ret = dmirror_check_atomic(dmirror, cmd.addr, 1343 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1344 break; 1345 1346 case HMM_DMIRROR_SNAPSHOT: 1347 ret = dmirror_snapshot(dmirror, &cmd); 1348 break; 1349 1350 case HMM_DMIRROR_RELEASE: 1351 dmirror_device_remove_chunks(dmirror->mdevice); 1352 ret = 0; 1353 break; 1354 1355 default: 1356 return -EINVAL; 1357 } 1358 if (ret) 1359 return ret; 1360 1361 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1362 return -EFAULT; 1363 1364 return 0; 1365 } 1366 1367 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1368 { 1369 unsigned long addr; 1370 1371 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1372 struct page *page; 1373 int ret; 1374 1375 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1376 if (!page) 1377 return -ENOMEM; 1378 1379 ret = vm_insert_page(vma, addr, page); 1380 if (ret) { 1381 __free_page(page); 1382 return ret; 1383 } 1384 put_page(page); 1385 } 1386 1387 return 0; 1388 } 1389 1390 static const struct file_operations dmirror_fops = { 1391 .open = dmirror_fops_open, 1392 .release = dmirror_fops_release, 1393 .mmap = dmirror_fops_mmap, 1394 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1395 .llseek = default_llseek, 1396 .owner = THIS_MODULE, 1397 }; 1398 1399 static void dmirror_devmem_free(struct page *page) 1400 { 1401 struct page *rpage = BACKING_PAGE(page); 1402 struct dmirror_device *mdevice; 1403 1404 if (rpage != page) 1405 __free_page(rpage); 1406 1407 mdevice = dmirror_page_to_device(page); 1408 spin_lock(&mdevice->lock); 1409 1410 /* Return page to our allocator if not freeing the chunk */ 1411 if (!dmirror_page_to_chunk(page)->remove) { 1412 mdevice->cfree++; 1413 page->zone_device_data = mdevice->free_pages; 1414 mdevice->free_pages = page; 1415 } 1416 spin_unlock(&mdevice->lock); 1417 } 1418 1419 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1420 { 1421 struct migrate_vma args = { 0 }; 1422 unsigned long src_pfns = 0; 1423 unsigned long dst_pfns = 0; 1424 struct page *rpage; 1425 struct dmirror *dmirror; 1426 vm_fault_t ret; 1427 1428 /* 1429 * Normally, a device would use the page->zone_device_data to point to 1430 * the mirror but here we use it to hold the page for the simulated 1431 * device memory and that page holds the pointer to the mirror. 1432 */ 1433 rpage = vmf->page->zone_device_data; 1434 dmirror = rpage->zone_device_data; 1435 1436 /* FIXME demonstrate how we can adjust migrate range */ 1437 args.vma = vmf->vma; 1438 args.start = vmf->address; 1439 args.end = args.start + PAGE_SIZE; 1440 args.src = &src_pfns; 1441 args.dst = &dst_pfns; 1442 args.pgmap_owner = dmirror->mdevice; 1443 args.flags = dmirror_select_device(dmirror); 1444 args.fault_page = vmf->page; 1445 1446 if (migrate_vma_setup(&args)) 1447 return VM_FAULT_SIGBUS; 1448 1449 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1450 if (ret) 1451 return ret; 1452 migrate_vma_pages(&args); 1453 /* 1454 * No device finalize step is needed since 1455 * dmirror_devmem_fault_alloc_and_copy() will have already 1456 * invalidated the device page table. 1457 */ 1458 migrate_vma_finalize(&args); 1459 return 0; 1460 } 1461 1462 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1463 .page_free = dmirror_devmem_free, 1464 .migrate_to_ram = dmirror_devmem_fault, 1465 }; 1466 1467 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1468 { 1469 dev_t dev; 1470 int ret; 1471 1472 dev = MKDEV(MAJOR(dmirror_dev), id); 1473 mutex_init(&mdevice->devmem_lock); 1474 spin_lock_init(&mdevice->lock); 1475 1476 cdev_init(&mdevice->cdevice, &dmirror_fops); 1477 mdevice->cdevice.owner = THIS_MODULE; 1478 device_initialize(&mdevice->device); 1479 mdevice->device.devt = dev; 1480 1481 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); 1482 if (ret) 1483 return ret; 1484 1485 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); 1486 if (ret) 1487 return ret; 1488 1489 /* Build a list of free ZONE_DEVICE struct pages */ 1490 return dmirror_allocate_chunk(mdevice, NULL); 1491 } 1492 1493 static void dmirror_device_remove(struct dmirror_device *mdevice) 1494 { 1495 dmirror_device_remove_chunks(mdevice); 1496 cdev_device_del(&mdevice->cdevice, &mdevice->device); 1497 } 1498 1499 static int __init hmm_dmirror_init(void) 1500 { 1501 int ret; 1502 int id = 0; 1503 int ndevices = 0; 1504 1505 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1506 "HMM_DMIRROR"); 1507 if (ret) 1508 goto err_unreg; 1509 1510 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1511 dmirror_devices[ndevices++].zone_device_type = 1512 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1513 dmirror_devices[ndevices++].zone_device_type = 1514 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1515 if (spm_addr_dev0 && spm_addr_dev1) { 1516 dmirror_devices[ndevices++].zone_device_type = 1517 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1518 dmirror_devices[ndevices++].zone_device_type = 1519 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1520 } 1521 for (id = 0; id < ndevices; id++) { 1522 ret = dmirror_device_init(dmirror_devices + id, id); 1523 if (ret) 1524 goto err_chrdev; 1525 } 1526 1527 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1528 return 0; 1529 1530 err_chrdev: 1531 while (--id >= 0) 1532 dmirror_device_remove(dmirror_devices + id); 1533 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1534 err_unreg: 1535 return ret; 1536 } 1537 1538 static void __exit hmm_dmirror_exit(void) 1539 { 1540 int id; 1541 1542 for (id = 0; id < DMIRROR_NDEVICES; id++) 1543 if (dmirror_devices[id].zone_device_type) 1544 dmirror_device_remove(dmirror_devices + id); 1545 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1546 } 1547 1548 module_init(hmm_dmirror_init); 1549 module_exit(hmm_dmirror_exit); 1550 MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module"); 1551 MODULE_LICENSE("GPL"); 1552