1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2133ff0eaSJérôme Glisse /* 3133ff0eaSJérôme Glisse * Copyright 2013 Red Hat Inc. 4133ff0eaSJérôme Glisse * 5f813f219SJérôme Glisse * Authors: Jérôme Glisse <jglisse@redhat.com> 6133ff0eaSJérôme Glisse */ 7133ff0eaSJérôme Glisse /* 8133ff0eaSJérôme Glisse * Refer to include/linux/hmm.h for information about heterogeneous memory 9133ff0eaSJérôme Glisse * management or HMM for short. 10133ff0eaSJérôme Glisse */ 11a520110eSChristoph Hellwig #include <linux/pagewalk.h> 12133ff0eaSJérôme Glisse #include <linux/hmm.h> 13858b54daSJérôme Glisse #include <linux/init.h> 14da4c3c73SJérôme Glisse #include <linux/rmap.h> 15da4c3c73SJérôme Glisse #include <linux/swap.h> 16133ff0eaSJérôme Glisse #include <linux/slab.h> 17133ff0eaSJérôme Glisse #include <linux/sched.h> 184ef589dcSJérôme Glisse #include <linux/mmzone.h> 194ef589dcSJérôme Glisse #include <linux/pagemap.h> 20da4c3c73SJérôme Glisse #include <linux/swapops.h> 21da4c3c73SJérôme Glisse #include <linux/hugetlb.h> 224ef589dcSJérôme Glisse #include <linux/memremap.h> 23c8a53b2dSJason Gunthorpe #include <linux/sched/mm.h> 247b2d55d2SJérôme Glisse #include <linux/jump_label.h> 2555c0ece8SJérôme Glisse #include <linux/dma-mapping.h> 26c0b12405SJérôme Glisse #include <linux/mmu_notifier.h> 274ef589dcSJérôme Glisse #include <linux/memory_hotplug.h> 284ef589dcSJérôme Glisse 2974eee180SJérôme Glisse struct hmm_vma_walk { 3074eee180SJérôme Glisse struct hmm_range *range; 31992de9a8SJérôme Glisse struct dev_pagemap *pgmap; 3274eee180SJérôme Glisse unsigned long last; 339a4903e4SChristoph Hellwig unsigned int flags; 3474eee180SJérôme Glisse }; 3574eee180SJérôme Glisse 362aee09d8SJérôme Glisse static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr, 372aee09d8SJérôme Glisse bool write_fault, uint64_t *pfn) 3874eee180SJérôme Glisse { 399b1ae605SKuehling, Felix unsigned int flags = FAULT_FLAG_REMOTE; 4074eee180SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 41f88a1e90SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 4274eee180SJérôme Glisse struct vm_area_struct *vma = walk->vma; 4350a7ca3cSSouptick Joarder vm_fault_t ret; 4474eee180SJérôme Glisse 456c64f2bbSRalph Campbell if (!vma) 466c64f2bbSRalph Campbell goto err; 476c64f2bbSRalph Campbell 489a4903e4SChristoph Hellwig if (hmm_vma_walk->flags & HMM_FAULT_ALLOW_RETRY) 499a4903e4SChristoph Hellwig flags |= FAULT_FLAG_ALLOW_RETRY; 509a4903e4SChristoph Hellwig if (write_fault) 519a4903e4SChristoph Hellwig flags |= FAULT_FLAG_WRITE; 529a4903e4SChristoph Hellwig 5350a7ca3cSSouptick Joarder ret = handle_mm_fault(vma, addr, flags); 54e709acccSJason Gunthorpe if (ret & VM_FAULT_RETRY) { 55e709acccSJason Gunthorpe /* Note, handle_mm_fault did up_read(&mm->mmap_sem)) */ 5673231612SJérôme Glisse return -EAGAIN; 57e709acccSJason Gunthorpe } 586c64f2bbSRalph Campbell if (ret & VM_FAULT_ERROR) 596c64f2bbSRalph Campbell goto err; 6074eee180SJérôme Glisse 6173231612SJérôme Glisse return -EBUSY; 626c64f2bbSRalph Campbell 636c64f2bbSRalph Campbell err: 646c64f2bbSRalph Campbell *pfn = range->values[HMM_PFN_ERROR]; 656c64f2bbSRalph Campbell return -EFAULT; 6674eee180SJérôme Glisse } 6774eee180SJérôme Glisse 68d28c2c9aSRalph Campbell static int hmm_pfns_fill(unsigned long addr, unsigned long end, 69d28c2c9aSRalph Campbell struct hmm_range *range, enum hmm_pfn_value_e value) 70da4c3c73SJérôme Glisse { 71ff05c0c6SJérôme Glisse uint64_t *pfns = range->pfns; 72da4c3c73SJérôme Glisse unsigned long i; 73da4c3c73SJérôme Glisse 74da4c3c73SJérôme Glisse i = (addr - range->start) >> PAGE_SHIFT; 75da4c3c73SJérôme Glisse for (; addr < end; addr += PAGE_SIZE, i++) 76d28c2c9aSRalph Campbell pfns[i] = range->values[value]; 77da4c3c73SJérôme Glisse 78da4c3c73SJérôme Glisse return 0; 79da4c3c73SJérôme Glisse } 80da4c3c73SJérôme Glisse 815504ed29SJérôme Glisse /* 82d2e8d551SRalph Campbell * hmm_vma_walk_hole_() - handle a range lacking valid pmd or pte(s) 83d2e8d551SRalph Campbell * @addr: range virtual start address (inclusive) 845504ed29SJérôme Glisse * @end: range virtual end address (exclusive) 852aee09d8SJérôme Glisse * @fault: should we fault or not ? 862aee09d8SJérôme Glisse * @write_fault: write fault ? 875504ed29SJérôme Glisse * @walk: mm_walk structure 88085ea250SRalph Campbell * Return: 0 on success, -EBUSY after page fault, or page fault error 895504ed29SJérôme Glisse * 905504ed29SJérôme Glisse * This function will be called whenever pmd_none() or pte_none() returns true, 915504ed29SJérôme Glisse * or whenever there is no page directory covering the virtual address range. 925504ed29SJérôme Glisse */ 932aee09d8SJérôme Glisse static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end, 942aee09d8SJérôme Glisse bool fault, bool write_fault, 95da4c3c73SJérôme Glisse struct mm_walk *walk) 96da4c3c73SJérôme Glisse { 9774eee180SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 9874eee180SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 99ff05c0c6SJérôme Glisse uint64_t *pfns = range->pfns; 1007f08263dSChristoph Hellwig unsigned long i; 101da4c3c73SJérôme Glisse 10274eee180SJérôme Glisse hmm_vma_walk->last = addr; 1037f08263dSChristoph Hellwig i = (addr - range->start) >> PAGE_SHIFT; 10463d5066fSJérôme Glisse 105c18ce674SRalph Campbell if (write_fault && walk->vma && !(walk->vma->vm_flags & VM_WRITE)) 106c18ce674SRalph Campbell return -EPERM; 107c18ce674SRalph Campbell 1087f08263dSChristoph Hellwig for (; addr < end; addr += PAGE_SIZE, i++) { 109f88a1e90SJérôme Glisse pfns[i] = range->values[HMM_PFN_NONE]; 1102aee09d8SJérôme Glisse if (fault || write_fault) { 11174eee180SJérôme Glisse int ret; 112da4c3c73SJérôme Glisse 1132aee09d8SJérôme Glisse ret = hmm_vma_do_fault(walk, addr, write_fault, 1142aee09d8SJérôme Glisse &pfns[i]); 11573231612SJérôme Glisse if (ret != -EBUSY) 11674eee180SJérôme Glisse return ret; 11774eee180SJérôme Glisse } 11874eee180SJérôme Glisse } 11974eee180SJérôme Glisse 12073231612SJérôme Glisse return (fault || write_fault) ? -EBUSY : 0; 1212aee09d8SJérôme Glisse } 1222aee09d8SJérôme Glisse 1232aee09d8SJérôme Glisse static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, 1242aee09d8SJérôme Glisse uint64_t pfns, uint64_t cpu_flags, 1252aee09d8SJérôme Glisse bool *fault, bool *write_fault) 1262aee09d8SJérôme Glisse { 127f88a1e90SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 128f88a1e90SJérôme Glisse 129d45d464bSChristoph Hellwig if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) 1302aee09d8SJérôme Glisse return; 1312aee09d8SJérôme Glisse 132023a019aSJérôme Glisse /* 133023a019aSJérôme Glisse * So we not only consider the individual per page request we also 134023a019aSJérôme Glisse * consider the default flags requested for the range. The API can 135d2e8d551SRalph Campbell * be used 2 ways. The first one where the HMM user coalesces 136d2e8d551SRalph Campbell * multiple page faults into one request and sets flags per pfn for 137d2e8d551SRalph Campbell * those faults. The second one where the HMM user wants to pre- 138023a019aSJérôme Glisse * fault a range with specific flags. For the latter one it is a 139023a019aSJérôme Glisse * waste to have the user pre-fill the pfn arrays with a default 140023a019aSJérôme Glisse * flags value. 141023a019aSJérôme Glisse */ 142023a019aSJérôme Glisse pfns = (pfns & range->pfn_flags_mask) | range->default_flags; 143023a019aSJérôme Glisse 1442aee09d8SJérôme Glisse /* We aren't ask to do anything ... */ 145f88a1e90SJérôme Glisse if (!(pfns & range->flags[HMM_PFN_VALID])) 1462aee09d8SJérôme Glisse return; 147d2e8d551SRalph Campbell /* If this is device memory then only fault if explicitly requested */ 148f88a1e90SJérôme Glisse if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) { 149f88a1e90SJérôme Glisse /* Do we fault on device memory ? */ 150f88a1e90SJérôme Glisse if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) { 151f88a1e90SJérôme Glisse *write_fault = pfns & range->flags[HMM_PFN_WRITE]; 152f88a1e90SJérôme Glisse *fault = true; 153f88a1e90SJérôme Glisse } 1542aee09d8SJérôme Glisse return; 1552aee09d8SJérôme Glisse } 156f88a1e90SJérôme Glisse 157f88a1e90SJérôme Glisse /* If CPU page table is not valid then we need to fault */ 158f88a1e90SJérôme Glisse *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]); 159f88a1e90SJérôme Glisse /* Need to write fault ? */ 160f88a1e90SJérôme Glisse if ((pfns & range->flags[HMM_PFN_WRITE]) && 161f88a1e90SJérôme Glisse !(cpu_flags & range->flags[HMM_PFN_WRITE])) { 162f88a1e90SJérôme Glisse *write_fault = true; 1632aee09d8SJérôme Glisse *fault = true; 1642aee09d8SJérôme Glisse } 1652aee09d8SJérôme Glisse } 1662aee09d8SJérôme Glisse 1672aee09d8SJérôme Glisse static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk, 1682aee09d8SJérôme Glisse const uint64_t *pfns, unsigned long npages, 1692aee09d8SJérôme Glisse uint64_t cpu_flags, bool *fault, 1702aee09d8SJérôme Glisse bool *write_fault) 1712aee09d8SJérôme Glisse { 1722aee09d8SJérôme Glisse unsigned long i; 1732aee09d8SJérôme Glisse 174d45d464bSChristoph Hellwig if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) { 1752aee09d8SJérôme Glisse *fault = *write_fault = false; 1762aee09d8SJérôme Glisse return; 1772aee09d8SJérôme Glisse } 1782aee09d8SJérôme Glisse 179a3e0d41cSJérôme Glisse *fault = *write_fault = false; 1802aee09d8SJérôme Glisse for (i = 0; i < npages; ++i) { 1812aee09d8SJérôme Glisse hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags, 1822aee09d8SJérôme Glisse fault, write_fault); 183a3e0d41cSJérôme Glisse if ((*write_fault)) 1842aee09d8SJérôme Glisse return; 1852aee09d8SJérôme Glisse } 1862aee09d8SJérôme Glisse } 1872aee09d8SJérôme Glisse 1882aee09d8SJérôme Glisse static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, 189b7a16c7aSSteven Price __always_unused int depth, struct mm_walk *walk) 1902aee09d8SJérôme Glisse { 1912aee09d8SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 1922aee09d8SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 1932aee09d8SJérôme Glisse bool fault, write_fault; 1942aee09d8SJérôme Glisse unsigned long i, npages; 1952aee09d8SJérôme Glisse uint64_t *pfns; 1962aee09d8SJérôme Glisse 1972aee09d8SJérôme Glisse i = (addr - range->start) >> PAGE_SHIFT; 1982aee09d8SJérôme Glisse npages = (end - addr) >> PAGE_SHIFT; 1992aee09d8SJérôme Glisse pfns = &range->pfns[i]; 2002aee09d8SJérôme Glisse hmm_range_need_fault(hmm_vma_walk, pfns, npages, 2012aee09d8SJérôme Glisse 0, &fault, &write_fault); 2022aee09d8SJérôme Glisse return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); 2032aee09d8SJérôme Glisse } 2042aee09d8SJérôme Glisse 205f88a1e90SJérôme Glisse static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd) 2062aee09d8SJérôme Glisse { 2072aee09d8SJérôme Glisse if (pmd_protnone(pmd)) 2082aee09d8SJérôme Glisse return 0; 209f88a1e90SJérôme Glisse return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] | 210f88a1e90SJérôme Glisse range->flags[HMM_PFN_WRITE] : 211f88a1e90SJérôme Glisse range->flags[HMM_PFN_VALID]; 212da4c3c73SJérôme Glisse } 213da4c3c73SJérôme Glisse 214992de9a8SJérôme Glisse #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2159d3973d6SChristoph Hellwig static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, 2169d3973d6SChristoph Hellwig unsigned long end, uint64_t *pfns, pmd_t pmd) 2179d3973d6SChristoph Hellwig { 21853f5c3f4SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 219f88a1e90SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 2202aee09d8SJérôme Glisse unsigned long pfn, npages, i; 2212aee09d8SJérôme Glisse bool fault, write_fault; 222f88a1e90SJérôme Glisse uint64_t cpu_flags; 22353f5c3f4SJérôme Glisse 2242aee09d8SJérôme Glisse npages = (end - addr) >> PAGE_SHIFT; 225f88a1e90SJérôme Glisse cpu_flags = pmd_to_hmm_pfn_flags(range, pmd); 2262aee09d8SJérôme Glisse hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags, 2272aee09d8SJérôme Glisse &fault, &write_fault); 22853f5c3f4SJérôme Glisse 2292aee09d8SJérôme Glisse if (pmd_protnone(pmd) || fault || write_fault) 2302aee09d8SJérôme Glisse return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); 23153f5c3f4SJérôme Glisse 232309f9a4fSChristoph Hellwig pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 233992de9a8SJérôme Glisse for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { 234992de9a8SJérôme Glisse if (pmd_devmap(pmd)) { 235992de9a8SJérôme Glisse hmm_vma_walk->pgmap = get_dev_pagemap(pfn, 236992de9a8SJérôme Glisse hmm_vma_walk->pgmap); 237992de9a8SJérôme Glisse if (unlikely(!hmm_vma_walk->pgmap)) 238992de9a8SJérôme Glisse return -EBUSY; 239992de9a8SJérôme Glisse } 240391aab11SJérôme Glisse pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags; 241992de9a8SJérôme Glisse } 242992de9a8SJérôme Glisse if (hmm_vma_walk->pgmap) { 243992de9a8SJérôme Glisse put_dev_pagemap(hmm_vma_walk->pgmap); 244992de9a8SJérôme Glisse hmm_vma_walk->pgmap = NULL; 245992de9a8SJérôme Glisse } 24653f5c3f4SJérôme Glisse hmm_vma_walk->last = end; 24753f5c3f4SJérôme Glisse return 0; 24853f5c3f4SJérôme Glisse } 2499d3973d6SChristoph Hellwig #else /* CONFIG_TRANSPARENT_HUGEPAGE */ 2509d3973d6SChristoph Hellwig /* stub to allow the code below to compile */ 2519d3973d6SChristoph Hellwig int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, 2529d3973d6SChristoph Hellwig unsigned long end, uint64_t *pfns, pmd_t pmd); 2539d3973d6SChristoph Hellwig #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 25453f5c3f4SJérôme Glisse 255f88a1e90SJérôme Glisse static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte) 2562aee09d8SJérôme Glisse { 257789c2af8SPhilip Yang if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte)) 2582aee09d8SJérôme Glisse return 0; 259f88a1e90SJérôme Glisse return pte_write(pte) ? range->flags[HMM_PFN_VALID] | 260f88a1e90SJérôme Glisse range->flags[HMM_PFN_WRITE] : 261f88a1e90SJérôme Glisse range->flags[HMM_PFN_VALID]; 2622aee09d8SJérôme Glisse } 2632aee09d8SJérôme Glisse 26453f5c3f4SJérôme Glisse static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, 26553f5c3f4SJérôme Glisse unsigned long end, pmd_t *pmdp, pte_t *ptep, 26653f5c3f4SJérôme Glisse uint64_t *pfn) 26753f5c3f4SJérôme Glisse { 26853f5c3f4SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 269f88a1e90SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 2702aee09d8SJérôme Glisse bool fault, write_fault; 2712aee09d8SJérôme Glisse uint64_t cpu_flags; 27253f5c3f4SJérôme Glisse pte_t pte = *ptep; 273f88a1e90SJérôme Glisse uint64_t orig_pfn = *pfn; 27453f5c3f4SJérôme Glisse 275f88a1e90SJérôme Glisse *pfn = range->values[HMM_PFN_NONE]; 27673231612SJérôme Glisse fault = write_fault = false; 27753f5c3f4SJérôme Glisse 27853f5c3f4SJérôme Glisse if (pte_none(pte)) { 27973231612SJérôme Glisse hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, 28073231612SJérôme Glisse &fault, &write_fault); 2812aee09d8SJérôme Glisse if (fault || write_fault) 28253f5c3f4SJérôme Glisse goto fault; 28353f5c3f4SJérôme Glisse return 0; 28453f5c3f4SJérôme Glisse } 28553f5c3f4SJérôme Glisse 28653f5c3f4SJérôme Glisse if (!pte_present(pte)) { 28753f5c3f4SJérôme Glisse swp_entry_t entry = pte_to_swp_entry(pte); 28853f5c3f4SJérôme Glisse 28953f5c3f4SJérôme Glisse if (!non_swap_entry(entry)) { 290e3fe8e55SYang, Philip cpu_flags = pte_to_hmm_pfn_flags(range, pte); 291e3fe8e55SYang, Philip hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, 292e3fe8e55SYang, Philip &fault, &write_fault); 2932aee09d8SJérôme Glisse if (fault || write_fault) 29453f5c3f4SJérôme Glisse goto fault; 29553f5c3f4SJérôme Glisse return 0; 29653f5c3f4SJérôme Glisse } 29753f5c3f4SJérôme Glisse 29853f5c3f4SJérôme Glisse /* 29953f5c3f4SJérôme Glisse * This is a special swap entry, ignore migration, use 30053f5c3f4SJérôme Glisse * device and report anything else as error. 30153f5c3f4SJérôme Glisse */ 30253f5c3f4SJérôme Glisse if (is_device_private_entry(entry)) { 303f88a1e90SJérôme Glisse cpu_flags = range->flags[HMM_PFN_VALID] | 304f88a1e90SJérôme Glisse range->flags[HMM_PFN_DEVICE_PRIVATE]; 3052aee09d8SJérôme Glisse cpu_flags |= is_write_device_private_entry(entry) ? 306f88a1e90SJérôme Glisse range->flags[HMM_PFN_WRITE] : 0; 307f88a1e90SJérôme Glisse hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, 308f88a1e90SJérôme Glisse &fault, &write_fault); 309f88a1e90SJérôme Glisse if (fault || write_fault) 310f88a1e90SJérôme Glisse goto fault; 311391aab11SJérôme Glisse *pfn = hmm_device_entry_from_pfn(range, 312391aab11SJérôme Glisse swp_offset(entry)); 313f88a1e90SJérôme Glisse *pfn |= cpu_flags; 31453f5c3f4SJérôme Glisse return 0; 31553f5c3f4SJérôme Glisse } 31653f5c3f4SJérôme Glisse 31753f5c3f4SJérôme Glisse if (is_migration_entry(entry)) { 3182aee09d8SJérôme Glisse if (fault || write_fault) { 31953f5c3f4SJérôme Glisse pte_unmap(ptep); 32053f5c3f4SJérôme Glisse hmm_vma_walk->last = addr; 321d2e8d551SRalph Campbell migration_entry_wait(walk->mm, pmdp, addr); 32273231612SJérôme Glisse return -EBUSY; 32353f5c3f4SJérôme Glisse } 32453f5c3f4SJérôme Glisse return 0; 32553f5c3f4SJérôme Glisse } 32653f5c3f4SJérôme Glisse 32753f5c3f4SJérôme Glisse /* Report error for everything else */ 328dfdc2207SJason Gunthorpe pte_unmap(ptep); 329f88a1e90SJérôme Glisse *pfn = range->values[HMM_PFN_ERROR]; 33053f5c3f4SJérôme Glisse return -EFAULT; 33173231612SJérôme Glisse } else { 33273231612SJérôme Glisse cpu_flags = pte_to_hmm_pfn_flags(range, pte); 33373231612SJérôme Glisse hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, 33473231612SJérôme Glisse &fault, &write_fault); 33553f5c3f4SJérôme Glisse } 33653f5c3f4SJérôme Glisse 3372aee09d8SJérôme Glisse if (fault || write_fault) 33853f5c3f4SJérôme Glisse goto fault; 33953f5c3f4SJérôme Glisse 340992de9a8SJérôme Glisse if (pte_devmap(pte)) { 341992de9a8SJérôme Glisse hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte), 342992de9a8SJérôme Glisse hmm_vma_walk->pgmap); 343dfdc2207SJason Gunthorpe if (unlikely(!hmm_vma_walk->pgmap)) { 344dfdc2207SJason Gunthorpe pte_unmap(ptep); 345992de9a8SJérôme Glisse return -EBUSY; 346dfdc2207SJason Gunthorpe } 347992de9a8SJérôme Glisse } else if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pte_special(pte)) { 348ac541f25SRalph Campbell if (!is_zero_pfn(pte_pfn(pte))) { 349dfdc2207SJason Gunthorpe pte_unmap(ptep); 350992de9a8SJérôme Glisse *pfn = range->values[HMM_PFN_SPECIAL]; 351992de9a8SJérôme Glisse return -EFAULT; 352992de9a8SJérôme Glisse } 353ac541f25SRalph Campbell /* 354ac541f25SRalph Campbell * Since each architecture defines a struct page for the zero 355ac541f25SRalph Campbell * page, just fall through and treat it like a normal page. 356ac541f25SRalph Campbell */ 357ac541f25SRalph Campbell } 358992de9a8SJérôme Glisse 359391aab11SJérôme Glisse *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags; 36053f5c3f4SJérôme Glisse return 0; 36153f5c3f4SJérôme Glisse 36253f5c3f4SJérôme Glisse fault: 363992de9a8SJérôme Glisse if (hmm_vma_walk->pgmap) { 364992de9a8SJérôme Glisse put_dev_pagemap(hmm_vma_walk->pgmap); 365992de9a8SJérôme Glisse hmm_vma_walk->pgmap = NULL; 366992de9a8SJérôme Glisse } 36753f5c3f4SJérôme Glisse pte_unmap(ptep); 36853f5c3f4SJérôme Glisse /* Fault any virtual address we were asked to fault */ 3692aee09d8SJérôme Glisse return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); 37053f5c3f4SJérôme Glisse } 37153f5c3f4SJérôme Glisse 372da4c3c73SJérôme Glisse static int hmm_vma_walk_pmd(pmd_t *pmdp, 373da4c3c73SJérôme Glisse unsigned long start, 374da4c3c73SJérôme Glisse unsigned long end, 375da4c3c73SJérôme Glisse struct mm_walk *walk) 376da4c3c73SJérôme Glisse { 37774eee180SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 37874eee180SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 379ff05c0c6SJérôme Glisse uint64_t *pfns = range->pfns; 380da4c3c73SJérôme Glisse unsigned long addr = start, i; 381da4c3c73SJérôme Glisse pte_t *ptep; 382da4c3c73SJérôme Glisse pmd_t pmd; 383da4c3c73SJérôme Glisse 384d08faca0SJérôme Glisse again: 385d08faca0SJérôme Glisse pmd = READ_ONCE(*pmdp); 386d08faca0SJérôme Glisse if (pmd_none(pmd)) 387b7a16c7aSSteven Price return hmm_vma_walk_hole(start, end, -1, walk); 388d08faca0SJérôme Glisse 389d08faca0SJérôme Glisse if (thp_migration_supported() && is_pmd_migration_entry(pmd)) { 390d08faca0SJérôme Glisse bool fault, write_fault; 391d08faca0SJérôme Glisse unsigned long npages; 392d08faca0SJérôme Glisse uint64_t *pfns; 393d08faca0SJérôme Glisse 394d08faca0SJérôme Glisse i = (addr - range->start) >> PAGE_SHIFT; 395d08faca0SJérôme Glisse npages = (end - addr) >> PAGE_SHIFT; 396d08faca0SJérôme Glisse pfns = &range->pfns[i]; 397d08faca0SJérôme Glisse 398d08faca0SJérôme Glisse hmm_range_need_fault(hmm_vma_walk, pfns, npages, 399d08faca0SJérôme Glisse 0, &fault, &write_fault); 400d08faca0SJérôme Glisse if (fault || write_fault) { 401d08faca0SJérôme Glisse hmm_vma_walk->last = addr; 402d2e8d551SRalph Campbell pmd_migration_entry_wait(walk->mm, pmdp); 40373231612SJérôme Glisse return -EBUSY; 404d08faca0SJérôme Glisse } 405*7d082987SJason Gunthorpe return hmm_pfns_fill(start, end, range, HMM_PFN_NONE); 406d08faca0SJérôme Glisse } else if (!pmd_present(pmd)) 407d28c2c9aSRalph Campbell return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); 408d08faca0SJérôme Glisse 409d08faca0SJérôme Glisse if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) { 410da4c3c73SJérôme Glisse /* 411d2e8d551SRalph Campbell * No need to take pmd_lock here, even if some other thread 412da4c3c73SJérôme Glisse * is splitting the huge pmd we will get that event through 413da4c3c73SJérôme Glisse * mmu_notifier callback. 414da4c3c73SJérôme Glisse * 415d2e8d551SRalph Campbell * So just read pmd value and check again it's a transparent 416da4c3c73SJérôme Glisse * huge or device mapping one and compute corresponding pfn 417da4c3c73SJérôme Glisse * values. 418da4c3c73SJérôme Glisse */ 419da4c3c73SJérôme Glisse pmd = pmd_read_atomic(pmdp); 420da4c3c73SJérôme Glisse barrier(); 421da4c3c73SJérôme Glisse if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd)) 422da4c3c73SJérôme Glisse goto again; 423da4c3c73SJérôme Glisse 424d08faca0SJérôme Glisse i = (addr - range->start) >> PAGE_SHIFT; 42553f5c3f4SJérôme Glisse return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd); 426da4c3c73SJérôme Glisse } 427da4c3c73SJérôme Glisse 428d08faca0SJérôme Glisse /* 429d2e8d551SRalph Campbell * We have handled all the valid cases above ie either none, migration, 430d08faca0SJérôme Glisse * huge or transparent huge. At this point either it is a valid pmd 431d08faca0SJérôme Glisse * entry pointing to pte directory or it is a bad pmd that will not 432d08faca0SJérôme Glisse * recover. 433d08faca0SJérôme Glisse */ 434d08faca0SJérôme Glisse if (pmd_bad(pmd)) 435d28c2c9aSRalph Campbell return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); 436da4c3c73SJérôme Glisse 437da4c3c73SJérôme Glisse ptep = pte_offset_map(pmdp, addr); 438d08faca0SJérôme Glisse i = (addr - range->start) >> PAGE_SHIFT; 439da4c3c73SJérôme Glisse for (; addr < end; addr += PAGE_SIZE, ptep++, i++) { 44053f5c3f4SJérôme Glisse int r; 441da4c3c73SJérôme Glisse 44253f5c3f4SJérôme Glisse r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]); 44353f5c3f4SJérôme Glisse if (r) { 444dfdc2207SJason Gunthorpe /* hmm_vma_handle_pte() did pte_unmap() */ 44574eee180SJérôme Glisse hmm_vma_walk->last = addr; 44653f5c3f4SJérôme Glisse return r; 44774eee180SJérôme Glisse } 448da4c3c73SJérôme Glisse } 449992de9a8SJérôme Glisse if (hmm_vma_walk->pgmap) { 450992de9a8SJérôme Glisse /* 451992de9a8SJérôme Glisse * We do put_dev_pagemap() here and not in hmm_vma_handle_pte() 452992de9a8SJérôme Glisse * so that we can leverage get_dev_pagemap() optimization which 453992de9a8SJérôme Glisse * will not re-take a reference on a pgmap if we already have 454992de9a8SJérôme Glisse * one. 455992de9a8SJérôme Glisse */ 456992de9a8SJérôme Glisse put_dev_pagemap(hmm_vma_walk->pgmap); 457992de9a8SJérôme Glisse hmm_vma_walk->pgmap = NULL; 458992de9a8SJérôme Glisse } 459da4c3c73SJérôme Glisse pte_unmap(ptep - 1); 460da4c3c73SJérôme Glisse 46153f5c3f4SJérôme Glisse hmm_vma_walk->last = addr; 462da4c3c73SJérôme Glisse return 0; 463da4c3c73SJérôme Glisse } 464da4c3c73SJérôme Glisse 465f0b3c45cSChristoph Hellwig #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \ 466f0b3c45cSChristoph Hellwig defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 467f0b3c45cSChristoph Hellwig static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud) 468f0b3c45cSChristoph Hellwig { 469f0b3c45cSChristoph Hellwig if (!pud_present(pud)) 470f0b3c45cSChristoph Hellwig return 0; 471f0b3c45cSChristoph Hellwig return pud_write(pud) ? range->flags[HMM_PFN_VALID] | 472f0b3c45cSChristoph Hellwig range->flags[HMM_PFN_WRITE] : 473f0b3c45cSChristoph Hellwig range->flags[HMM_PFN_VALID]; 474f0b3c45cSChristoph Hellwig } 475f0b3c45cSChristoph Hellwig 476f0b3c45cSChristoph Hellwig static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, 477992de9a8SJérôme Glisse struct mm_walk *walk) 478992de9a8SJérôme Glisse { 479992de9a8SJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 480992de9a8SJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 4813afc4236SSteven Price unsigned long addr = start; 482992de9a8SJérôme Glisse pud_t pud; 4833afc4236SSteven Price int ret = 0; 4843afc4236SSteven Price spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma); 485992de9a8SJérôme Glisse 4863afc4236SSteven Price if (!ptl) 4873afc4236SSteven Price return 0; 4883afc4236SSteven Price 4893afc4236SSteven Price /* Normally we don't want to split the huge page */ 4903afc4236SSteven Price walk->action = ACTION_CONTINUE; 4913afc4236SSteven Price 492992de9a8SJérôme Glisse pud = READ_ONCE(*pudp); 4933afc4236SSteven Price if (pud_none(pud)) { 49405fc1df9SJason Gunthorpe spin_unlock(ptl); 49505fc1df9SJason Gunthorpe return hmm_vma_walk_hole(start, end, -1, walk); 4963afc4236SSteven Price } 497992de9a8SJérôme Glisse 498992de9a8SJérôme Glisse if (pud_huge(pud) && pud_devmap(pud)) { 499992de9a8SJérôme Glisse unsigned long i, npages, pfn; 500992de9a8SJérôme Glisse uint64_t *pfns, cpu_flags; 501992de9a8SJérôme Glisse bool fault, write_fault; 502992de9a8SJérôme Glisse 5033afc4236SSteven Price if (!pud_present(pud)) { 50405fc1df9SJason Gunthorpe spin_unlock(ptl); 50505fc1df9SJason Gunthorpe return hmm_vma_walk_hole(start, end, -1, walk); 5063afc4236SSteven Price } 507992de9a8SJérôme Glisse 508992de9a8SJérôme Glisse i = (addr - range->start) >> PAGE_SHIFT; 509992de9a8SJérôme Glisse npages = (end - addr) >> PAGE_SHIFT; 510992de9a8SJérôme Glisse pfns = &range->pfns[i]; 511992de9a8SJérôme Glisse 512992de9a8SJérôme Glisse cpu_flags = pud_to_hmm_pfn_flags(range, pud); 513992de9a8SJérôme Glisse hmm_range_need_fault(hmm_vma_walk, pfns, npages, 514992de9a8SJérôme Glisse cpu_flags, &fault, &write_fault); 5153afc4236SSteven Price if (fault || write_fault) { 51605fc1df9SJason Gunthorpe spin_unlock(ptl); 51705fc1df9SJason Gunthorpe return hmm_vma_walk_hole_(addr, end, fault, write_fault, 51805fc1df9SJason Gunthorpe walk); 5193afc4236SSteven Price } 520992de9a8SJérôme Glisse 521992de9a8SJérôme Glisse pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 522992de9a8SJérôme Glisse for (i = 0; i < npages; ++i, ++pfn) { 523992de9a8SJérôme Glisse hmm_vma_walk->pgmap = get_dev_pagemap(pfn, 524992de9a8SJérôme Glisse hmm_vma_walk->pgmap); 5253afc4236SSteven Price if (unlikely(!hmm_vma_walk->pgmap)) { 5263afc4236SSteven Price ret = -EBUSY; 5273afc4236SSteven Price goto out_unlock; 5283afc4236SSteven Price } 529391aab11SJérôme Glisse pfns[i] = hmm_device_entry_from_pfn(range, pfn) | 530391aab11SJérôme Glisse cpu_flags; 531992de9a8SJérôme Glisse } 532992de9a8SJérôme Glisse if (hmm_vma_walk->pgmap) { 533992de9a8SJérôme Glisse put_dev_pagemap(hmm_vma_walk->pgmap); 534992de9a8SJérôme Glisse hmm_vma_walk->pgmap = NULL; 535992de9a8SJérôme Glisse } 536992de9a8SJérôme Glisse hmm_vma_walk->last = end; 5373afc4236SSteven Price goto out_unlock; 538992de9a8SJérôme Glisse } 539992de9a8SJérôme Glisse 5403afc4236SSteven Price /* Ask for the PUD to be split */ 5413afc4236SSteven Price walk->action = ACTION_SUBTREE; 542992de9a8SJérôme Glisse 5433afc4236SSteven Price out_unlock: 5443afc4236SSteven Price spin_unlock(ptl); 545992de9a8SJérôme Glisse return ret; 546992de9a8SJérôme Glisse } 547f0b3c45cSChristoph Hellwig #else 548f0b3c45cSChristoph Hellwig #define hmm_vma_walk_pud NULL 549f0b3c45cSChristoph Hellwig #endif 550992de9a8SJérôme Glisse 551251bbe59SChristoph Hellwig #ifdef CONFIG_HUGETLB_PAGE 55263d5066fSJérôme Glisse static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, 55363d5066fSJérôme Glisse unsigned long start, unsigned long end, 55463d5066fSJérôme Glisse struct mm_walk *walk) 55563d5066fSJérôme Glisse { 55605c23af4SChristoph Hellwig unsigned long addr = start, i, pfn; 55763d5066fSJérôme Glisse struct hmm_vma_walk *hmm_vma_walk = walk->private; 55863d5066fSJérôme Glisse struct hmm_range *range = hmm_vma_walk->range; 55963d5066fSJérôme Glisse struct vm_area_struct *vma = walk->vma; 56063d5066fSJérôme Glisse uint64_t orig_pfn, cpu_flags; 56163d5066fSJérôme Glisse bool fault, write_fault; 56263d5066fSJérôme Glisse spinlock_t *ptl; 56363d5066fSJérôme Glisse pte_t entry; 56463d5066fSJérôme Glisse int ret = 0; 56563d5066fSJérôme Glisse 566d2e8d551SRalph Campbell ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte); 56763d5066fSJérôme Glisse entry = huge_ptep_get(pte); 56863d5066fSJérôme Glisse 5697f08263dSChristoph Hellwig i = (start - range->start) >> PAGE_SHIFT; 57063d5066fSJérôme Glisse orig_pfn = range->pfns[i]; 57163d5066fSJérôme Glisse range->pfns[i] = range->values[HMM_PFN_NONE]; 57263d5066fSJérôme Glisse cpu_flags = pte_to_hmm_pfn_flags(range, entry); 57363d5066fSJérôme Glisse fault = write_fault = false; 57463d5066fSJérôme Glisse hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, 57563d5066fSJérôme Glisse &fault, &write_fault); 57663d5066fSJérôme Glisse if (fault || write_fault) { 57763d5066fSJérôme Glisse ret = -ENOENT; 57863d5066fSJérôme Glisse goto unlock; 57963d5066fSJérôme Glisse } 58063d5066fSJérôme Glisse 58105c23af4SChristoph Hellwig pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); 5827f08263dSChristoph Hellwig for (; addr < end; addr += PAGE_SIZE, i++, pfn++) 583391aab11SJérôme Glisse range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) | 584391aab11SJérôme Glisse cpu_flags; 58563d5066fSJérôme Glisse hmm_vma_walk->last = end; 58663d5066fSJérôme Glisse 58763d5066fSJérôme Glisse unlock: 58863d5066fSJérôme Glisse spin_unlock(ptl); 58963d5066fSJérôme Glisse 59063d5066fSJérôme Glisse if (ret == -ENOENT) 59163d5066fSJérôme Glisse return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); 59263d5066fSJérôme Glisse 59363d5066fSJérôme Glisse return ret; 59463d5066fSJérôme Glisse } 595251bbe59SChristoph Hellwig #else 596251bbe59SChristoph Hellwig #define hmm_vma_walk_hugetlb_entry NULL 597251bbe59SChristoph Hellwig #endif /* CONFIG_HUGETLB_PAGE */ 59863d5066fSJérôme Glisse 599d28c2c9aSRalph Campbell static int hmm_vma_walk_test(unsigned long start, unsigned long end, 600d28c2c9aSRalph Campbell struct mm_walk *walk) 60133cd47dcSJérôme Glisse { 602d28c2c9aSRalph Campbell struct hmm_vma_walk *hmm_vma_walk = walk->private; 603d28c2c9aSRalph Campbell struct hmm_range *range = hmm_vma_walk->range; 604d28c2c9aSRalph Campbell struct vm_area_struct *vma = walk->vma; 605d28c2c9aSRalph Campbell 606d28c2c9aSRalph Campbell /* 607d28c2c9aSRalph Campbell * Skip vma ranges that don't have struct page backing them or 608d28c2c9aSRalph Campbell * map I/O devices directly. 609d28c2c9aSRalph Campbell */ 610d28c2c9aSRalph Campbell if (vma->vm_flags & (VM_IO | VM_PFNMAP | VM_MIXEDMAP)) 611d28c2c9aSRalph Campbell return -EFAULT; 612d28c2c9aSRalph Campbell 613d28c2c9aSRalph Campbell /* 614d28c2c9aSRalph Campbell * If the vma does not allow read access, then assume that it does not 615d28c2c9aSRalph Campbell * allow write access either. HMM does not support architectures 616d28c2c9aSRalph Campbell * that allow write without read. 617d28c2c9aSRalph Campbell */ 618d28c2c9aSRalph Campbell if (!(vma->vm_flags & VM_READ)) { 619d28c2c9aSRalph Campbell bool fault, write_fault; 620d28c2c9aSRalph Campbell 621d28c2c9aSRalph Campbell /* 622d28c2c9aSRalph Campbell * Check to see if a fault is requested for any page in the 623d28c2c9aSRalph Campbell * range. 624d28c2c9aSRalph Campbell */ 625d28c2c9aSRalph Campbell hmm_range_need_fault(hmm_vma_walk, range->pfns + 626d28c2c9aSRalph Campbell ((start - range->start) >> PAGE_SHIFT), 627d28c2c9aSRalph Campbell (end - start) >> PAGE_SHIFT, 628d28c2c9aSRalph Campbell 0, &fault, &write_fault); 629d28c2c9aSRalph Campbell if (fault || write_fault) 630d28c2c9aSRalph Campbell return -EFAULT; 631d28c2c9aSRalph Campbell 632d28c2c9aSRalph Campbell hmm_pfns_fill(start, end, range, HMM_PFN_NONE); 633d28c2c9aSRalph Campbell hmm_vma_walk->last = end; 634d28c2c9aSRalph Campbell 635d28c2c9aSRalph Campbell /* Skip this vma and continue processing the next vma. */ 636d28c2c9aSRalph Campbell return 1; 637d28c2c9aSRalph Campbell } 638d28c2c9aSRalph Campbell 639d28c2c9aSRalph Campbell return 0; 64033cd47dcSJérôme Glisse } 64133cd47dcSJérôme Glisse 6427b86ac33SChristoph Hellwig static const struct mm_walk_ops hmm_walk_ops = { 6437b86ac33SChristoph Hellwig .pud_entry = hmm_vma_walk_pud, 6447b86ac33SChristoph Hellwig .pmd_entry = hmm_vma_walk_pmd, 6457b86ac33SChristoph Hellwig .pte_hole = hmm_vma_walk_hole, 6467b86ac33SChristoph Hellwig .hugetlb_entry = hmm_vma_walk_hugetlb_entry, 647d28c2c9aSRalph Campbell .test_walk = hmm_vma_walk_test, 6487b86ac33SChristoph Hellwig }; 6497b86ac33SChristoph Hellwig 6509a4903e4SChristoph Hellwig /** 6519a4903e4SChristoph Hellwig * hmm_range_fault - try to fault some address in a virtual address range 65208232a45SJérôme Glisse * @range: range being faulted 6539a4903e4SChristoph Hellwig * @flags: HMM_FAULT_* flags 65473231612SJérôme Glisse * 6559a4903e4SChristoph Hellwig * Return: the number of valid pages in range->pfns[] (from range start 6569a4903e4SChristoph Hellwig * address), which may be zero. On error one of the following status codes 6579a4903e4SChristoph Hellwig * can be returned: 6589a4903e4SChristoph Hellwig * 6599a4903e4SChristoph Hellwig * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma 6609a4903e4SChristoph Hellwig * (e.g., device file vma). 66173231612SJérôme Glisse * -ENOMEM: Out of memory. 6629a4903e4SChristoph Hellwig * -EPERM: Invalid permission (e.g., asking for write and range is read 6639a4903e4SChristoph Hellwig * only). 6649a4903e4SChristoph Hellwig * -EAGAIN: A page fault needs to be retried and mmap_sem was dropped. 6659a4903e4SChristoph Hellwig * -EBUSY: The range has been invalidated and the caller needs to wait for 6669a4903e4SChristoph Hellwig * the invalidation to finish. 6679a4903e4SChristoph Hellwig * -EFAULT: Invalid (i.e., either no valid vma or it is illegal to access 6689a4903e4SChristoph Hellwig * that range) number of valid pages in range->pfns[] (from 66973231612SJérôme Glisse * range start address). 67074eee180SJérôme Glisse * 67174eee180SJérôme Glisse * This is similar to a regular CPU page fault except that it will not trigger 67273231612SJérôme Glisse * any memory migration if the memory being faulted is not accessible by CPUs 67373231612SJérôme Glisse * and caller does not ask for migration. 67474eee180SJérôme Glisse * 675ff05c0c6SJérôme Glisse * On error, for one virtual address in the range, the function will mark the 676ff05c0c6SJérôme Glisse * corresponding HMM pfn entry with an error flag. 67774eee180SJérôme Glisse */ 6789a4903e4SChristoph Hellwig long hmm_range_fault(struct hmm_range *range, unsigned int flags) 67974eee180SJérôme Glisse { 680d28c2c9aSRalph Campbell struct hmm_vma_walk hmm_vma_walk = { 681d28c2c9aSRalph Campbell .range = range, 682d28c2c9aSRalph Campbell .last = range->start, 683d28c2c9aSRalph Campbell .flags = flags, 684d28c2c9aSRalph Campbell }; 685a22dd506SJason Gunthorpe struct mm_struct *mm = range->notifier->mm; 68674eee180SJérôme Glisse int ret; 68774eee180SJérôme Glisse 68804ec32fbSJason Gunthorpe lockdep_assert_held(&mm->mmap_sem); 689a3e0d41cSJérôme Glisse 690a3e0d41cSJérôme Glisse do { 691a3e0d41cSJérôme Glisse /* If range is no longer valid force retry. */ 692a22dd506SJason Gunthorpe if (mmu_interval_check_retry(range->notifier, 693a22dd506SJason Gunthorpe range->notifier_seq)) 6942bcbeaefSChristoph Hellwig return -EBUSY; 695d28c2c9aSRalph Campbell ret = walk_page_range(mm, hmm_vma_walk.last, range->end, 6967b86ac33SChristoph Hellwig &hmm_walk_ops, &hmm_vma_walk); 697d28c2c9aSRalph Campbell } while (ret == -EBUSY); 698a3e0d41cSJérôme Glisse 699d28c2c9aSRalph Campbell if (ret) 70073231612SJérôme Glisse return ret; 70173231612SJérôme Glisse return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT; 70274eee180SJérôme Glisse } 70373231612SJérôme Glisse EXPORT_SYMBOL(hmm_range_fault); 704