1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This is a module to test the HMM (Heterogeneous Memory Management) 4 * mirror and zone device private memory migration APIs of the kernel. 5 * Userspace programs can register with the driver to mirror their own address 6 * space and can use the device to read/write any valid virtual address. 7 */ 8 #include <linux/init.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/memremap.h> 16 #include <linux/mutex.h> 17 #include <linux/rwsem.h> 18 #include <linux/sched.h> 19 #include <linux/slab.h> 20 #include <linux/highmem.h> 21 #include <linux/delay.h> 22 #include <linux/pagemap.h> 23 #include <linux/hmm.h> 24 #include <linux/vmalloc.h> 25 #include <linux/swap.h> 26 #include <linux/swapops.h> 27 #include <linux/sched/mm.h> 28 #include <linux/platform_device.h> 29 #include <linux/rmap.h> 30 #include <linux/mmu_notifier.h> 31 #include <linux/migrate.h> 32 33 #include "test_hmm_uapi.h" 34 35 #define DMIRROR_NDEVICES 4 36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) 38 #define DEVMEM_CHUNKS_RESERVE 16 39 40 /* 41 * For device_private pages, dpage is just a dummy struct page 42 * representing a piece of device memory. dmirror_devmem_alloc_page 43 * allocates a real system memory page as backing storage to fake a 44 * real device. zone_device_data points to that backing page. But 45 * for device_coherent memory, the struct page represents real 46 * physical CPU-accessible memory that we can use directly. 47 */ 48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ 49 (page)->zone_device_data : (page)) 50 51 static unsigned long spm_addr_dev0; 52 module_param(spm_addr_dev0, long, 0644); 53 MODULE_PARM_DESC(spm_addr_dev0, 54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 55 56 static unsigned long spm_addr_dev1; 57 module_param(spm_addr_dev1, long, 0644); 58 MODULE_PARM_DESC(spm_addr_dev1, 59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); 60 61 static const struct dev_pagemap_ops dmirror_devmem_ops; 62 static const struct mmu_interval_notifier_ops dmirror_min_ops; 63 static dev_t dmirror_dev; 64 65 struct dmirror_device; 66 67 struct dmirror_bounce { 68 void *ptr; 69 unsigned long size; 70 unsigned long addr; 71 unsigned long cpages; 72 }; 73 74 #define DPT_XA_TAG_ATOMIC 1UL 75 #define DPT_XA_TAG_WRITE 3UL 76 77 /* 78 * Data structure to track address ranges and register for mmu interval 79 * notifier updates. 80 */ 81 struct dmirror_interval { 82 struct mmu_interval_notifier notifier; 83 struct dmirror *dmirror; 84 }; 85 86 /* 87 * Data attached to the open device file. 88 * Note that it might be shared after a fork(). 89 */ 90 struct dmirror { 91 struct dmirror_device *mdevice; 92 struct xarray pt; 93 struct mmu_interval_notifier notifier; 94 struct mutex mutex; 95 }; 96 97 /* 98 * ZONE_DEVICE pages for migration and simulating device memory. 99 */ 100 struct dmirror_chunk { 101 struct dev_pagemap pagemap; 102 struct dmirror_device *mdevice; 103 }; 104 105 /* 106 * Per device data. 107 */ 108 struct dmirror_device { 109 struct cdev cdevice; 110 struct hmm_devmem *devmem; 111 unsigned int zone_device_type; 112 113 unsigned int devmem_capacity; 114 unsigned int devmem_count; 115 struct dmirror_chunk **devmem_chunks; 116 struct mutex devmem_lock; /* protects the above */ 117 118 unsigned long calloc; 119 unsigned long cfree; 120 struct page *free_pages; 121 spinlock_t lock; /* protects the above */ 122 }; 123 124 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; 125 126 static int dmirror_bounce_init(struct dmirror_bounce *bounce, 127 unsigned long addr, 128 unsigned long size) 129 { 130 bounce->addr = addr; 131 bounce->size = size; 132 bounce->cpages = 0; 133 bounce->ptr = vmalloc(size); 134 if (!bounce->ptr) 135 return -ENOMEM; 136 return 0; 137 } 138 139 static bool dmirror_is_private_zone(struct dmirror_device *mdevice) 140 { 141 return (mdevice->zone_device_type == 142 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? true : false; 143 } 144 145 static enum migrate_vma_direction 146 dmirror_select_device(struct dmirror *dmirror) 147 { 148 return (dmirror->mdevice->zone_device_type == 149 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? 150 MIGRATE_VMA_SELECT_DEVICE_PRIVATE : 151 MIGRATE_VMA_SELECT_DEVICE_COHERENT; 152 } 153 154 static void dmirror_bounce_fini(struct dmirror_bounce *bounce) 155 { 156 vfree(bounce->ptr); 157 } 158 159 static int dmirror_fops_open(struct inode *inode, struct file *filp) 160 { 161 struct cdev *cdev = inode->i_cdev; 162 struct dmirror *dmirror; 163 int ret; 164 165 /* Mirror this process address space */ 166 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); 167 if (dmirror == NULL) 168 return -ENOMEM; 169 170 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); 171 mutex_init(&dmirror->mutex); 172 xa_init(&dmirror->pt); 173 174 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 175 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); 176 if (ret) { 177 kfree(dmirror); 178 return ret; 179 } 180 181 filp->private_data = dmirror; 182 return 0; 183 } 184 185 static int dmirror_fops_release(struct inode *inode, struct file *filp) 186 { 187 struct dmirror *dmirror = filp->private_data; 188 189 mmu_interval_notifier_remove(&dmirror->notifier); 190 xa_destroy(&dmirror->pt); 191 kfree(dmirror); 192 return 0; 193 } 194 195 static struct dmirror_device *dmirror_page_to_device(struct page *page) 196 197 { 198 return container_of(page->pgmap, struct dmirror_chunk, 199 pagemap)->mdevice; 200 } 201 202 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) 203 { 204 unsigned long *pfns = range->hmm_pfns; 205 unsigned long pfn; 206 207 for (pfn = (range->start >> PAGE_SHIFT); 208 pfn < (range->end >> PAGE_SHIFT); 209 pfn++, pfns++) { 210 struct page *page; 211 void *entry; 212 213 /* 214 * Since we asked for hmm_range_fault() to populate pages, 215 * it shouldn't return an error entry on success. 216 */ 217 WARN_ON(*pfns & HMM_PFN_ERROR); 218 WARN_ON(!(*pfns & HMM_PFN_VALID)); 219 220 page = hmm_pfn_to_page(*pfns); 221 WARN_ON(!page); 222 223 entry = page; 224 if (*pfns & HMM_PFN_WRITE) 225 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 226 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) 227 return -EFAULT; 228 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 229 if (xa_is_err(entry)) 230 return xa_err(entry); 231 } 232 233 return 0; 234 } 235 236 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, 237 unsigned long end) 238 { 239 unsigned long pfn; 240 void *entry; 241 242 /* 243 * The XArray doesn't hold references to pages since it relies on 244 * the mmu notifier to clear page pointers when they become stale. 245 * Therefore, it is OK to just clear the entry. 246 */ 247 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, 248 end >> PAGE_SHIFT) 249 xa_erase(&dmirror->pt, pfn); 250 } 251 252 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, 253 const struct mmu_notifier_range *range, 254 unsigned long cur_seq) 255 { 256 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); 257 258 /* 259 * Ignore invalidation callbacks for device private pages since 260 * the invalidation is handled as part of the migration process. 261 */ 262 if (range->event == MMU_NOTIFY_MIGRATE && 263 range->owner == dmirror->mdevice) 264 return true; 265 266 if (mmu_notifier_range_blockable(range)) 267 mutex_lock(&dmirror->mutex); 268 else if (!mutex_trylock(&dmirror->mutex)) 269 return false; 270 271 mmu_interval_set_seq(mni, cur_seq); 272 dmirror_do_update(dmirror, range->start, range->end); 273 274 mutex_unlock(&dmirror->mutex); 275 return true; 276 } 277 278 static const struct mmu_interval_notifier_ops dmirror_min_ops = { 279 .invalidate = dmirror_interval_invalidate, 280 }; 281 282 static int dmirror_range_fault(struct dmirror *dmirror, 283 struct hmm_range *range) 284 { 285 struct mm_struct *mm = dmirror->notifier.mm; 286 unsigned long timeout = 287 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 288 int ret; 289 290 while (true) { 291 if (time_after(jiffies, timeout)) { 292 ret = -EBUSY; 293 goto out; 294 } 295 296 range->notifier_seq = mmu_interval_read_begin(range->notifier); 297 mmap_read_lock(mm); 298 ret = hmm_range_fault(range); 299 mmap_read_unlock(mm); 300 if (ret) { 301 if (ret == -EBUSY) 302 continue; 303 goto out; 304 } 305 306 mutex_lock(&dmirror->mutex); 307 if (mmu_interval_read_retry(range->notifier, 308 range->notifier_seq)) { 309 mutex_unlock(&dmirror->mutex); 310 continue; 311 } 312 break; 313 } 314 315 ret = dmirror_do_fault(dmirror, range); 316 317 mutex_unlock(&dmirror->mutex); 318 out: 319 return ret; 320 } 321 322 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, 323 unsigned long end, bool write) 324 { 325 struct mm_struct *mm = dmirror->notifier.mm; 326 unsigned long addr; 327 unsigned long pfns[64]; 328 struct hmm_range range = { 329 .notifier = &dmirror->notifier, 330 .hmm_pfns = pfns, 331 .pfn_flags_mask = 0, 332 .default_flags = 333 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), 334 .dev_private_owner = dmirror->mdevice, 335 }; 336 int ret = 0; 337 338 /* Since the mm is for the mirrored process, get a reference first. */ 339 if (!mmget_not_zero(mm)) 340 return 0; 341 342 for (addr = start; addr < end; addr = range.end) { 343 range.start = addr; 344 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 345 346 ret = dmirror_range_fault(dmirror, &range); 347 if (ret) 348 break; 349 } 350 351 mmput(mm); 352 return ret; 353 } 354 355 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, 356 unsigned long end, struct dmirror_bounce *bounce) 357 { 358 unsigned long pfn; 359 void *ptr; 360 361 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 362 363 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 364 void *entry; 365 struct page *page; 366 void *tmp; 367 368 entry = xa_load(&dmirror->pt, pfn); 369 page = xa_untag_pointer(entry); 370 if (!page) 371 return -ENOENT; 372 373 tmp = kmap(page); 374 memcpy(ptr, tmp, PAGE_SIZE); 375 kunmap(page); 376 377 ptr += PAGE_SIZE; 378 bounce->cpages++; 379 } 380 381 return 0; 382 } 383 384 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 385 { 386 struct dmirror_bounce bounce; 387 unsigned long start, end; 388 unsigned long size = cmd->npages << PAGE_SHIFT; 389 int ret; 390 391 start = cmd->addr; 392 end = start + size; 393 if (end < start) 394 return -EINVAL; 395 396 ret = dmirror_bounce_init(&bounce, start, size); 397 if (ret) 398 return ret; 399 400 while (1) { 401 mutex_lock(&dmirror->mutex); 402 ret = dmirror_do_read(dmirror, start, end, &bounce); 403 mutex_unlock(&dmirror->mutex); 404 if (ret != -ENOENT) 405 break; 406 407 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 408 ret = dmirror_fault(dmirror, start, end, false); 409 if (ret) 410 break; 411 cmd->faults++; 412 } 413 414 if (ret == 0) { 415 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 416 bounce.size)) 417 ret = -EFAULT; 418 } 419 cmd->cpages = bounce.cpages; 420 dmirror_bounce_fini(&bounce); 421 return ret; 422 } 423 424 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, 425 unsigned long end, struct dmirror_bounce *bounce) 426 { 427 unsigned long pfn; 428 void *ptr; 429 430 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); 431 432 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 433 void *entry; 434 struct page *page; 435 void *tmp; 436 437 entry = xa_load(&dmirror->pt, pfn); 438 page = xa_untag_pointer(entry); 439 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE) 440 return -ENOENT; 441 442 tmp = kmap(page); 443 memcpy(tmp, ptr, PAGE_SIZE); 444 kunmap(page); 445 446 ptr += PAGE_SIZE; 447 bounce->cpages++; 448 } 449 450 return 0; 451 } 452 453 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) 454 { 455 struct dmirror_bounce bounce; 456 unsigned long start, end; 457 unsigned long size = cmd->npages << PAGE_SHIFT; 458 int ret; 459 460 start = cmd->addr; 461 end = start + size; 462 if (end < start) 463 return -EINVAL; 464 465 ret = dmirror_bounce_init(&bounce, start, size); 466 if (ret) 467 return ret; 468 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr), 469 bounce.size)) { 470 ret = -EFAULT; 471 goto fini; 472 } 473 474 while (1) { 475 mutex_lock(&dmirror->mutex); 476 ret = dmirror_do_write(dmirror, start, end, &bounce); 477 mutex_unlock(&dmirror->mutex); 478 if (ret != -ENOENT) 479 break; 480 481 start = cmd->addr + (bounce.cpages << PAGE_SHIFT); 482 ret = dmirror_fault(dmirror, start, end, true); 483 if (ret) 484 break; 485 cmd->faults++; 486 } 487 488 fini: 489 cmd->cpages = bounce.cpages; 490 dmirror_bounce_fini(&bounce); 491 return ret; 492 } 493 494 static int dmirror_allocate_chunk(struct dmirror_device *mdevice, 495 struct page **ppage) 496 { 497 struct dmirror_chunk *devmem; 498 struct resource *res = NULL; 499 unsigned long pfn; 500 unsigned long pfn_first; 501 unsigned long pfn_last; 502 void *ptr; 503 int ret = -ENOMEM; 504 505 devmem = kzalloc(sizeof(*devmem), GFP_KERNEL); 506 if (!devmem) 507 return ret; 508 509 switch (mdevice->zone_device_type) { 510 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE: 511 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE, 512 "hmm_dmirror"); 513 if (IS_ERR_OR_NULL(res)) 514 goto err_devmem; 515 devmem->pagemap.range.start = res->start; 516 devmem->pagemap.range.end = res->end; 517 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 518 break; 519 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: 520 devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? 521 spm_addr_dev0 : 522 spm_addr_dev1; 523 devmem->pagemap.range.end = devmem->pagemap.range.start + 524 DEVMEM_CHUNK_SIZE - 1; 525 devmem->pagemap.type = MEMORY_DEVICE_COHERENT; 526 break; 527 default: 528 ret = -EINVAL; 529 goto err_devmem; 530 } 531 532 devmem->pagemap.nr_range = 1; 533 devmem->pagemap.ops = &dmirror_devmem_ops; 534 devmem->pagemap.owner = mdevice; 535 536 mutex_lock(&mdevice->devmem_lock); 537 538 if (mdevice->devmem_count == mdevice->devmem_capacity) { 539 struct dmirror_chunk **new_chunks; 540 unsigned int new_capacity; 541 542 new_capacity = mdevice->devmem_capacity + 543 DEVMEM_CHUNKS_RESERVE; 544 new_chunks = krealloc(mdevice->devmem_chunks, 545 sizeof(new_chunks[0]) * new_capacity, 546 GFP_KERNEL); 547 if (!new_chunks) 548 goto err_release; 549 mdevice->devmem_capacity = new_capacity; 550 mdevice->devmem_chunks = new_chunks; 551 } 552 ptr = memremap_pages(&devmem->pagemap, numa_node_id()); 553 if (IS_ERR_OR_NULL(ptr)) { 554 if (ptr) 555 ret = PTR_ERR(ptr); 556 else 557 ret = -EFAULT; 558 goto err_release; 559 } 560 561 devmem->mdevice = mdevice; 562 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT; 563 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT); 564 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem; 565 566 mutex_unlock(&mdevice->devmem_lock); 567 568 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n", 569 DEVMEM_CHUNK_SIZE / (1024 * 1024), 570 mdevice->devmem_count, 571 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)), 572 pfn_first, pfn_last); 573 574 spin_lock(&mdevice->lock); 575 for (pfn = pfn_first; pfn < pfn_last; pfn++) { 576 struct page *page = pfn_to_page(pfn); 577 578 page->zone_device_data = mdevice->free_pages; 579 mdevice->free_pages = page; 580 } 581 if (ppage) { 582 *ppage = mdevice->free_pages; 583 mdevice->free_pages = (*ppage)->zone_device_data; 584 mdevice->calloc++; 585 } 586 spin_unlock(&mdevice->lock); 587 588 return 0; 589 590 err_release: 591 mutex_unlock(&mdevice->devmem_lock); 592 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 593 release_mem_region(devmem->pagemap.range.start, 594 range_len(&devmem->pagemap.range)); 595 err_devmem: 596 kfree(devmem); 597 598 return ret; 599 } 600 601 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) 602 { 603 struct page *dpage = NULL; 604 struct page *rpage = NULL; 605 606 /* 607 * For ZONE_DEVICE private type, this is a fake device so we allocate 608 * real system memory to store our device memory. 609 * For ZONE_DEVICE coherent type we use the actual dpage to store the 610 * data and ignore rpage. 611 */ 612 if (dmirror_is_private_zone(mdevice)) { 613 rpage = alloc_page(GFP_HIGHUSER); 614 if (!rpage) 615 return NULL; 616 } 617 spin_lock(&mdevice->lock); 618 619 if (mdevice->free_pages) { 620 dpage = mdevice->free_pages; 621 mdevice->free_pages = dpage->zone_device_data; 622 mdevice->calloc++; 623 spin_unlock(&mdevice->lock); 624 } else { 625 spin_unlock(&mdevice->lock); 626 if (dmirror_allocate_chunk(mdevice, &dpage)) 627 goto error; 628 } 629 630 dpage->zone_device_data = rpage; 631 lock_page(dpage); 632 return dpage; 633 634 error: 635 if (rpage) 636 __free_page(rpage); 637 return NULL; 638 } 639 640 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, 641 struct dmirror *dmirror) 642 { 643 struct dmirror_device *mdevice = dmirror->mdevice; 644 const unsigned long *src = args->src; 645 unsigned long *dst = args->dst; 646 unsigned long addr; 647 648 for (addr = args->start; addr < args->end; addr += PAGE_SIZE, 649 src++, dst++) { 650 struct page *spage; 651 struct page *dpage; 652 struct page *rpage; 653 654 if (!(*src & MIGRATE_PFN_MIGRATE)) 655 continue; 656 657 /* 658 * Note that spage might be NULL which is OK since it is an 659 * unallocated pte_none() or read-only zero page. 660 */ 661 spage = migrate_pfn_to_page(*src); 662 if (WARN(spage && is_zone_device_page(spage), 663 "page already in device spage pfn: 0x%lx\n", 664 page_to_pfn(spage))) 665 continue; 666 667 dpage = dmirror_devmem_alloc_page(mdevice); 668 if (!dpage) 669 continue; 670 671 rpage = BACKING_PAGE(dpage); 672 if (spage) 673 copy_highpage(rpage, spage); 674 else 675 clear_highpage(rpage); 676 677 /* 678 * Normally, a device would use the page->zone_device_data to 679 * point to the mirror but here we use it to hold the page for 680 * the simulated device memory and that page holds the pointer 681 * to the mirror. 682 */ 683 rpage->zone_device_data = dmirror; 684 685 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", 686 page_to_pfn(spage), page_to_pfn(dpage)); 687 *dst = migrate_pfn(page_to_pfn(dpage)); 688 if ((*src & MIGRATE_PFN_WRITE) || 689 (!spage && args->vma->vm_flags & VM_WRITE)) 690 *dst |= MIGRATE_PFN_WRITE; 691 } 692 } 693 694 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, 695 unsigned long end) 696 { 697 unsigned long pfn; 698 699 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { 700 void *entry; 701 702 entry = xa_load(&dmirror->pt, pfn); 703 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) 704 return -EPERM; 705 } 706 707 return 0; 708 } 709 710 static int dmirror_atomic_map(unsigned long start, unsigned long end, 711 struct page **pages, struct dmirror *dmirror) 712 { 713 unsigned long pfn, mapped = 0; 714 int i; 715 716 /* Map the migrated pages into the device's page tables. */ 717 mutex_lock(&dmirror->mutex); 718 719 for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) { 720 void *entry; 721 722 if (!pages[i]) 723 continue; 724 725 entry = pages[i]; 726 entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); 727 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 728 if (xa_is_err(entry)) { 729 mutex_unlock(&dmirror->mutex); 730 return xa_err(entry); 731 } 732 733 mapped++; 734 } 735 736 mutex_unlock(&dmirror->mutex); 737 return mapped; 738 } 739 740 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, 741 struct dmirror *dmirror) 742 { 743 unsigned long start = args->start; 744 unsigned long end = args->end; 745 const unsigned long *src = args->src; 746 const unsigned long *dst = args->dst; 747 unsigned long pfn; 748 749 /* Map the migrated pages into the device's page tables. */ 750 mutex_lock(&dmirror->mutex); 751 752 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, 753 src++, dst++) { 754 struct page *dpage; 755 void *entry; 756 757 if (!(*src & MIGRATE_PFN_MIGRATE)) 758 continue; 759 760 dpage = migrate_pfn_to_page(*dst); 761 if (!dpage) 762 continue; 763 764 entry = BACKING_PAGE(dpage); 765 if (*dst & MIGRATE_PFN_WRITE) 766 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); 767 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); 768 if (xa_is_err(entry)) { 769 mutex_unlock(&dmirror->mutex); 770 return xa_err(entry); 771 } 772 } 773 774 mutex_unlock(&dmirror->mutex); 775 return 0; 776 } 777 778 static int dmirror_exclusive(struct dmirror *dmirror, 779 struct hmm_dmirror_cmd *cmd) 780 { 781 unsigned long start, end, addr; 782 unsigned long size = cmd->npages << PAGE_SHIFT; 783 struct mm_struct *mm = dmirror->notifier.mm; 784 struct page *pages[64]; 785 struct dmirror_bounce bounce; 786 unsigned long next; 787 int ret; 788 789 start = cmd->addr; 790 end = start + size; 791 if (end < start) 792 return -EINVAL; 793 794 /* Since the mm is for the mirrored process, get a reference first. */ 795 if (!mmget_not_zero(mm)) 796 return -EINVAL; 797 798 mmap_read_lock(mm); 799 for (addr = start; addr < end; addr = next) { 800 unsigned long mapped = 0; 801 int i; 802 803 if (end < addr + (ARRAY_SIZE(pages) << PAGE_SHIFT)) 804 next = end; 805 else 806 next = addr + (ARRAY_SIZE(pages) << PAGE_SHIFT); 807 808 ret = make_device_exclusive_range(mm, addr, next, pages, NULL); 809 /* 810 * Do dmirror_atomic_map() iff all pages are marked for 811 * exclusive access to avoid accessing uninitialized 812 * fields of pages. 813 */ 814 if (ret == (next - addr) >> PAGE_SHIFT) 815 mapped = dmirror_atomic_map(addr, next, pages, dmirror); 816 for (i = 0; i < ret; i++) { 817 if (pages[i]) { 818 unlock_page(pages[i]); 819 put_page(pages[i]); 820 } 821 } 822 823 if (addr + (mapped << PAGE_SHIFT) < next) { 824 mmap_read_unlock(mm); 825 mmput(mm); 826 return -EBUSY; 827 } 828 } 829 mmap_read_unlock(mm); 830 mmput(mm); 831 832 /* Return the migrated data for verification. */ 833 ret = dmirror_bounce_init(&bounce, start, size); 834 if (ret) 835 return ret; 836 mutex_lock(&dmirror->mutex); 837 ret = dmirror_do_read(dmirror, start, end, &bounce); 838 mutex_unlock(&dmirror->mutex); 839 if (ret == 0) { 840 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 841 bounce.size)) 842 ret = -EFAULT; 843 } 844 845 cmd->cpages = bounce.cpages; 846 dmirror_bounce_fini(&bounce); 847 return ret; 848 } 849 850 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, 851 struct dmirror *dmirror) 852 { 853 const unsigned long *src = args->src; 854 unsigned long *dst = args->dst; 855 unsigned long start = args->start; 856 unsigned long end = args->end; 857 unsigned long addr; 858 859 for (addr = start; addr < end; addr += PAGE_SIZE, 860 src++, dst++) { 861 struct page *dpage, *spage; 862 863 spage = migrate_pfn_to_page(*src); 864 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) 865 continue; 866 867 if (WARN_ON(!is_device_private_page(spage) && 868 !is_device_coherent_page(spage))) 869 continue; 870 spage = BACKING_PAGE(spage); 871 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); 872 if (!dpage) 873 continue; 874 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", 875 page_to_pfn(spage), page_to_pfn(dpage)); 876 877 lock_page(dpage); 878 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); 879 copy_highpage(dpage, spage); 880 *dst = migrate_pfn(page_to_pfn(dpage)); 881 if (*src & MIGRATE_PFN_WRITE) 882 *dst |= MIGRATE_PFN_WRITE; 883 } 884 return 0; 885 } 886 887 static unsigned long 888 dmirror_successful_migrated_pages(struct migrate_vma *migrate) 889 { 890 unsigned long cpages = 0; 891 unsigned long i; 892 893 for (i = 0; i < migrate->npages; i++) { 894 if (migrate->src[i] & MIGRATE_PFN_VALID && 895 migrate->src[i] & MIGRATE_PFN_MIGRATE) 896 cpages++; 897 } 898 return cpages; 899 } 900 901 static int dmirror_migrate_to_system(struct dmirror *dmirror, 902 struct hmm_dmirror_cmd *cmd) 903 { 904 unsigned long start, end, addr; 905 unsigned long size = cmd->npages << PAGE_SHIFT; 906 struct mm_struct *mm = dmirror->notifier.mm; 907 struct vm_area_struct *vma; 908 unsigned long src_pfns[64] = { 0 }; 909 unsigned long dst_pfns[64] = { 0 }; 910 struct migrate_vma args; 911 unsigned long next; 912 int ret; 913 914 start = cmd->addr; 915 end = start + size; 916 if (end < start) 917 return -EINVAL; 918 919 /* Since the mm is for the mirrored process, get a reference first. */ 920 if (!mmget_not_zero(mm)) 921 return -EINVAL; 922 923 cmd->cpages = 0; 924 mmap_read_lock(mm); 925 for (addr = start; addr < end; addr = next) { 926 vma = vma_lookup(mm, addr); 927 if (!vma || !(vma->vm_flags & VM_READ)) { 928 ret = -EINVAL; 929 goto out; 930 } 931 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 932 if (next > vma->vm_end) 933 next = vma->vm_end; 934 935 args.vma = vma; 936 args.src = src_pfns; 937 args.dst = dst_pfns; 938 args.start = addr; 939 args.end = next; 940 args.pgmap_owner = dmirror->mdevice; 941 args.flags = dmirror_select_device(dmirror); 942 943 ret = migrate_vma_setup(&args); 944 if (ret) 945 goto out; 946 947 pr_debug("Migrating from device mem to sys mem\n"); 948 dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 949 950 migrate_vma_pages(&args); 951 cmd->cpages += dmirror_successful_migrated_pages(&args); 952 migrate_vma_finalize(&args); 953 } 954 out: 955 mmap_read_unlock(mm); 956 mmput(mm); 957 958 return ret; 959 } 960 961 static int dmirror_migrate_to_device(struct dmirror *dmirror, 962 struct hmm_dmirror_cmd *cmd) 963 { 964 unsigned long start, end, addr; 965 unsigned long size = cmd->npages << PAGE_SHIFT; 966 struct mm_struct *mm = dmirror->notifier.mm; 967 struct vm_area_struct *vma; 968 unsigned long src_pfns[64] = { 0 }; 969 unsigned long dst_pfns[64] = { 0 }; 970 struct dmirror_bounce bounce; 971 struct migrate_vma args; 972 unsigned long next; 973 int ret; 974 975 start = cmd->addr; 976 end = start + size; 977 if (end < start) 978 return -EINVAL; 979 980 /* Since the mm is for the mirrored process, get a reference first. */ 981 if (!mmget_not_zero(mm)) 982 return -EINVAL; 983 984 mmap_read_lock(mm); 985 for (addr = start; addr < end; addr = next) { 986 vma = vma_lookup(mm, addr); 987 if (!vma || !(vma->vm_flags & VM_READ)) { 988 ret = -EINVAL; 989 goto out; 990 } 991 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT)); 992 if (next > vma->vm_end) 993 next = vma->vm_end; 994 995 args.vma = vma; 996 args.src = src_pfns; 997 args.dst = dst_pfns; 998 args.start = addr; 999 args.end = next; 1000 args.pgmap_owner = dmirror->mdevice; 1001 args.flags = MIGRATE_VMA_SELECT_SYSTEM; 1002 ret = migrate_vma_setup(&args); 1003 if (ret) 1004 goto out; 1005 1006 pr_debug("Migrating from sys mem to device mem\n"); 1007 dmirror_migrate_alloc_and_copy(&args, dmirror); 1008 migrate_vma_pages(&args); 1009 dmirror_migrate_finalize_and_map(&args, dmirror); 1010 migrate_vma_finalize(&args); 1011 } 1012 mmap_read_unlock(mm); 1013 mmput(mm); 1014 1015 /* 1016 * Return the migrated data for verification. 1017 * Only for pages in device zone 1018 */ 1019 ret = dmirror_bounce_init(&bounce, start, size); 1020 if (ret) 1021 return ret; 1022 mutex_lock(&dmirror->mutex); 1023 ret = dmirror_do_read(dmirror, start, end, &bounce); 1024 mutex_unlock(&dmirror->mutex); 1025 if (ret == 0) { 1026 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, 1027 bounce.size)) 1028 ret = -EFAULT; 1029 } 1030 cmd->cpages = bounce.cpages; 1031 dmirror_bounce_fini(&bounce); 1032 return ret; 1033 1034 out: 1035 mmap_read_unlock(mm); 1036 mmput(mm); 1037 return ret; 1038 } 1039 1040 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, 1041 unsigned char *perm, unsigned long entry) 1042 { 1043 struct page *page; 1044 1045 if (entry & HMM_PFN_ERROR) { 1046 *perm = HMM_DMIRROR_PROT_ERROR; 1047 return; 1048 } 1049 if (!(entry & HMM_PFN_VALID)) { 1050 *perm = HMM_DMIRROR_PROT_NONE; 1051 return; 1052 } 1053 1054 page = hmm_pfn_to_page(entry); 1055 if (is_device_private_page(page)) { 1056 /* Is the page migrated to this device or some other? */ 1057 if (dmirror->mdevice == dmirror_page_to_device(page)) 1058 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL; 1059 else 1060 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE; 1061 } else if (is_device_coherent_page(page)) { 1062 /* Is the page migrated to this device or some other? */ 1063 if (dmirror->mdevice == dmirror_page_to_device(page)) 1064 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL; 1065 else 1066 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE; 1067 } else if (is_zero_pfn(page_to_pfn(page))) 1068 *perm = HMM_DMIRROR_PROT_ZERO; 1069 else 1070 *perm = HMM_DMIRROR_PROT_NONE; 1071 if (entry & HMM_PFN_WRITE) 1072 *perm |= HMM_DMIRROR_PROT_WRITE; 1073 else 1074 *perm |= HMM_DMIRROR_PROT_READ; 1075 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT) 1076 *perm |= HMM_DMIRROR_PROT_PMD; 1077 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT) 1078 *perm |= HMM_DMIRROR_PROT_PUD; 1079 } 1080 1081 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni, 1082 const struct mmu_notifier_range *range, 1083 unsigned long cur_seq) 1084 { 1085 struct dmirror_interval *dmi = 1086 container_of(mni, struct dmirror_interval, notifier); 1087 struct dmirror *dmirror = dmi->dmirror; 1088 1089 if (mmu_notifier_range_blockable(range)) 1090 mutex_lock(&dmirror->mutex); 1091 else if (!mutex_trylock(&dmirror->mutex)) 1092 return false; 1093 1094 /* 1095 * Snapshots only need to set the sequence number since any 1096 * invalidation in the interval invalidates the whole snapshot. 1097 */ 1098 mmu_interval_set_seq(mni, cur_seq); 1099 1100 mutex_unlock(&dmirror->mutex); 1101 return true; 1102 } 1103 1104 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = { 1105 .invalidate = dmirror_snapshot_invalidate, 1106 }; 1107 1108 static int dmirror_range_snapshot(struct dmirror *dmirror, 1109 struct hmm_range *range, 1110 unsigned char *perm) 1111 { 1112 struct mm_struct *mm = dmirror->notifier.mm; 1113 struct dmirror_interval notifier; 1114 unsigned long timeout = 1115 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1116 unsigned long i; 1117 unsigned long n; 1118 int ret = 0; 1119 1120 notifier.dmirror = dmirror; 1121 range->notifier = ¬ifier.notifier; 1122 1123 ret = mmu_interval_notifier_insert(range->notifier, mm, 1124 range->start, range->end - range->start, 1125 &dmirror_mrn_ops); 1126 if (ret) 1127 return ret; 1128 1129 while (true) { 1130 if (time_after(jiffies, timeout)) { 1131 ret = -EBUSY; 1132 goto out; 1133 } 1134 1135 range->notifier_seq = mmu_interval_read_begin(range->notifier); 1136 1137 mmap_read_lock(mm); 1138 ret = hmm_range_fault(range); 1139 mmap_read_unlock(mm); 1140 if (ret) { 1141 if (ret == -EBUSY) 1142 continue; 1143 goto out; 1144 } 1145 1146 mutex_lock(&dmirror->mutex); 1147 if (mmu_interval_read_retry(range->notifier, 1148 range->notifier_seq)) { 1149 mutex_unlock(&dmirror->mutex); 1150 continue; 1151 } 1152 break; 1153 } 1154 1155 n = (range->end - range->start) >> PAGE_SHIFT; 1156 for (i = 0; i < n; i++) 1157 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); 1158 1159 mutex_unlock(&dmirror->mutex); 1160 out: 1161 mmu_interval_notifier_remove(range->notifier); 1162 return ret; 1163 } 1164 1165 static int dmirror_snapshot(struct dmirror *dmirror, 1166 struct hmm_dmirror_cmd *cmd) 1167 { 1168 struct mm_struct *mm = dmirror->notifier.mm; 1169 unsigned long start, end; 1170 unsigned long size = cmd->npages << PAGE_SHIFT; 1171 unsigned long addr; 1172 unsigned long next; 1173 unsigned long pfns[64]; 1174 unsigned char perm[64]; 1175 char __user *uptr; 1176 struct hmm_range range = { 1177 .hmm_pfns = pfns, 1178 .dev_private_owner = dmirror->mdevice, 1179 }; 1180 int ret = 0; 1181 1182 start = cmd->addr; 1183 end = start + size; 1184 if (end < start) 1185 return -EINVAL; 1186 1187 /* Since the mm is for the mirrored process, get a reference first. */ 1188 if (!mmget_not_zero(mm)) 1189 return -EINVAL; 1190 1191 /* 1192 * Register a temporary notifier to detect invalidations even if it 1193 * overlaps with other mmu_interval_notifiers. 1194 */ 1195 uptr = u64_to_user_ptr(cmd->ptr); 1196 for (addr = start; addr < end; addr = next) { 1197 unsigned long n; 1198 1199 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); 1200 range.start = addr; 1201 range.end = next; 1202 1203 ret = dmirror_range_snapshot(dmirror, &range, perm); 1204 if (ret) 1205 break; 1206 1207 n = (range.end - range.start) >> PAGE_SHIFT; 1208 if (copy_to_user(uptr, perm, n)) { 1209 ret = -EFAULT; 1210 break; 1211 } 1212 1213 cmd->cpages += n; 1214 uptr += n; 1215 } 1216 mmput(mm); 1217 1218 return ret; 1219 } 1220 1221 static long dmirror_fops_unlocked_ioctl(struct file *filp, 1222 unsigned int command, 1223 unsigned long arg) 1224 { 1225 void __user *uarg = (void __user *)arg; 1226 struct hmm_dmirror_cmd cmd; 1227 struct dmirror *dmirror; 1228 int ret; 1229 1230 dmirror = filp->private_data; 1231 if (!dmirror) 1232 return -EINVAL; 1233 1234 if (copy_from_user(&cmd, uarg, sizeof(cmd))) 1235 return -EFAULT; 1236 1237 if (cmd.addr & ~PAGE_MASK) 1238 return -EINVAL; 1239 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT))) 1240 return -EINVAL; 1241 1242 cmd.cpages = 0; 1243 cmd.faults = 0; 1244 1245 switch (command) { 1246 case HMM_DMIRROR_READ: 1247 ret = dmirror_read(dmirror, &cmd); 1248 break; 1249 1250 case HMM_DMIRROR_WRITE: 1251 ret = dmirror_write(dmirror, &cmd); 1252 break; 1253 1254 case HMM_DMIRROR_MIGRATE_TO_DEV: 1255 ret = dmirror_migrate_to_device(dmirror, &cmd); 1256 break; 1257 1258 case HMM_DMIRROR_MIGRATE_TO_SYS: 1259 ret = dmirror_migrate_to_system(dmirror, &cmd); 1260 break; 1261 1262 case HMM_DMIRROR_EXCLUSIVE: 1263 ret = dmirror_exclusive(dmirror, &cmd); 1264 break; 1265 1266 case HMM_DMIRROR_CHECK_EXCLUSIVE: 1267 ret = dmirror_check_atomic(dmirror, cmd.addr, 1268 cmd.addr + (cmd.npages << PAGE_SHIFT)); 1269 break; 1270 1271 case HMM_DMIRROR_SNAPSHOT: 1272 ret = dmirror_snapshot(dmirror, &cmd); 1273 break; 1274 1275 default: 1276 return -EINVAL; 1277 } 1278 if (ret) 1279 return ret; 1280 1281 if (copy_to_user(uarg, &cmd, sizeof(cmd))) 1282 return -EFAULT; 1283 1284 return 0; 1285 } 1286 1287 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma) 1288 { 1289 unsigned long addr; 1290 1291 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { 1292 struct page *page; 1293 int ret; 1294 1295 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1296 if (!page) 1297 return -ENOMEM; 1298 1299 ret = vm_insert_page(vma, addr, page); 1300 if (ret) { 1301 __free_page(page); 1302 return ret; 1303 } 1304 put_page(page); 1305 } 1306 1307 return 0; 1308 } 1309 1310 static const struct file_operations dmirror_fops = { 1311 .open = dmirror_fops_open, 1312 .release = dmirror_fops_release, 1313 .mmap = dmirror_fops_mmap, 1314 .unlocked_ioctl = dmirror_fops_unlocked_ioctl, 1315 .llseek = default_llseek, 1316 .owner = THIS_MODULE, 1317 }; 1318 1319 static void dmirror_devmem_free(struct page *page) 1320 { 1321 struct page *rpage = BACKING_PAGE(page); 1322 struct dmirror_device *mdevice; 1323 1324 if (rpage != page) 1325 __free_page(rpage); 1326 1327 mdevice = dmirror_page_to_device(page); 1328 spin_lock(&mdevice->lock); 1329 mdevice->cfree++; 1330 page->zone_device_data = mdevice->free_pages; 1331 mdevice->free_pages = page; 1332 spin_unlock(&mdevice->lock); 1333 } 1334 1335 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) 1336 { 1337 struct migrate_vma args; 1338 unsigned long src_pfns = 0; 1339 unsigned long dst_pfns = 0; 1340 struct page *rpage; 1341 struct dmirror *dmirror; 1342 vm_fault_t ret; 1343 1344 /* 1345 * Normally, a device would use the page->zone_device_data to point to 1346 * the mirror but here we use it to hold the page for the simulated 1347 * device memory and that page holds the pointer to the mirror. 1348 */ 1349 rpage = vmf->page->zone_device_data; 1350 dmirror = rpage->zone_device_data; 1351 1352 /* FIXME demonstrate how we can adjust migrate range */ 1353 args.vma = vmf->vma; 1354 args.start = vmf->address; 1355 args.end = args.start + PAGE_SIZE; 1356 args.src = &src_pfns; 1357 args.dst = &dst_pfns; 1358 args.pgmap_owner = dmirror->mdevice; 1359 args.flags = dmirror_select_device(dmirror); 1360 1361 if (migrate_vma_setup(&args)) 1362 return VM_FAULT_SIGBUS; 1363 1364 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); 1365 if (ret) 1366 return ret; 1367 migrate_vma_pages(&args); 1368 /* 1369 * No device finalize step is needed since 1370 * dmirror_devmem_fault_alloc_and_copy() will have already 1371 * invalidated the device page table. 1372 */ 1373 migrate_vma_finalize(&args); 1374 return 0; 1375 } 1376 1377 static const struct dev_pagemap_ops dmirror_devmem_ops = { 1378 .page_free = dmirror_devmem_free, 1379 .migrate_to_ram = dmirror_devmem_fault, 1380 }; 1381 1382 static int dmirror_device_init(struct dmirror_device *mdevice, int id) 1383 { 1384 dev_t dev; 1385 int ret; 1386 1387 dev = MKDEV(MAJOR(dmirror_dev), id); 1388 mutex_init(&mdevice->devmem_lock); 1389 spin_lock_init(&mdevice->lock); 1390 1391 cdev_init(&mdevice->cdevice, &dmirror_fops); 1392 mdevice->cdevice.owner = THIS_MODULE; 1393 ret = cdev_add(&mdevice->cdevice, dev, 1); 1394 if (ret) 1395 return ret; 1396 1397 /* Build a list of free ZONE_DEVICE struct pages */ 1398 return dmirror_allocate_chunk(mdevice, NULL); 1399 } 1400 1401 static void dmirror_device_remove(struct dmirror_device *mdevice) 1402 { 1403 unsigned int i; 1404 1405 if (mdevice->devmem_chunks) { 1406 for (i = 0; i < mdevice->devmem_count; i++) { 1407 struct dmirror_chunk *devmem = 1408 mdevice->devmem_chunks[i]; 1409 1410 memunmap_pages(&devmem->pagemap); 1411 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) 1412 release_mem_region(devmem->pagemap.range.start, 1413 range_len(&devmem->pagemap.range)); 1414 kfree(devmem); 1415 } 1416 kfree(mdevice->devmem_chunks); 1417 } 1418 1419 cdev_del(&mdevice->cdevice); 1420 } 1421 1422 static int __init hmm_dmirror_init(void) 1423 { 1424 int ret; 1425 int id = 0; 1426 int ndevices = 0; 1427 1428 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES, 1429 "HMM_DMIRROR"); 1430 if (ret) 1431 goto err_unreg; 1432 1433 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0])); 1434 dmirror_devices[ndevices++].zone_device_type = 1435 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1436 dmirror_devices[ndevices++].zone_device_type = 1437 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE; 1438 if (spm_addr_dev0 && spm_addr_dev1) { 1439 dmirror_devices[ndevices++].zone_device_type = 1440 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1441 dmirror_devices[ndevices++].zone_device_type = 1442 HMM_DMIRROR_MEMORY_DEVICE_COHERENT; 1443 } 1444 for (id = 0; id < ndevices; id++) { 1445 ret = dmirror_device_init(dmirror_devices + id, id); 1446 if (ret) 1447 goto err_chrdev; 1448 } 1449 1450 pr_info("HMM test module loaded. This is only for testing HMM.\n"); 1451 return 0; 1452 1453 err_chrdev: 1454 while (--id >= 0) 1455 dmirror_device_remove(dmirror_devices + id); 1456 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1457 err_unreg: 1458 return ret; 1459 } 1460 1461 static void __exit hmm_dmirror_exit(void) 1462 { 1463 int id; 1464 1465 for (id = 0; id < DMIRROR_NDEVICES; id++) 1466 if (dmirror_devices[id].zone_device_type) 1467 dmirror_device_remove(dmirror_devices + id); 1468 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES); 1469 } 1470 1471 module_init(hmm_dmirror_init); 1472 module_exit(hmm_dmirror_exit); 1473 MODULE_LICENSE("GPL"); 1474