Lines Matching refs:migrate
25 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_skip() local
29 migrate->dst[migrate->npages] = 0; in migrate_vma_collect_skip()
30 migrate->src[migrate->npages++] = 0; in migrate_vma_collect_skip()
41 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_hole() local
49 (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) && in migrate_vma_collect_hole()
52 migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE | in migrate_vma_collect_hole()
54 migrate->dst[migrate->npages] = 0; in migrate_vma_collect_hole()
55 migrate->npages++; in migrate_vma_collect_hole()
56 migrate->cpages++; in migrate_vma_collect_hole()
66 migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE; in migrate_vma_collect_hole()
67 migrate->dst[migrate->npages] = 0; in migrate_vma_collect_hole()
68 migrate->npages++; in migrate_vma_collect_hole()
69 migrate->cpages++; in migrate_vma_collect_hole()
142 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_huge_pmd() local
154 if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { in migrate_vma_collect_huge_pmd()
172 !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || in migrate_vma_collect_huge_pmd()
173 (folio->pgmap->owner != migrate->pgmap_owner)) { in migrate_vma_collect_huge_pmd()
199 (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) && in migrate_vma_collect_huge_pmd()
212 migrate->src[migrate->npages] = migrate_pfn(pfn) | write in migrate_vma_collect_huge_pmd()
215 migrate->dst[migrate->npages++] = 0; in migrate_vma_collect_huge_pmd()
216 migrate->cpages++; in migrate_vma_collect_huge_pmd()
219 migrate->npages--; in migrate_vma_collect_huge_pmd()
220 migrate->cpages--; in migrate_vma_collect_huge_pmd()
221 migrate->src[migrate->npages] = 0; in migrate_vma_collect_huge_pmd()
222 migrate->dst[migrate->npages] = 0; in migrate_vma_collect_huge_pmd()
252 struct migrate_vma *migrate = walk->private; in migrate_vma_collect_pmd() local
257 struct folio *fault_folio = migrate->fault_page ? in migrate_vma_collect_pmd()
258 page_folio(migrate->fault_page) : NULL; in migrate_vma_collect_pmd()
290 migrate->cpages++; in migrate_vma_collect_pmd()
307 if (!(migrate->flags & in migrate_vma_collect_pmd()
309 pgmap->owner != migrate->pgmap_owner) in migrate_vma_collect_pmd()
319 migrate->fault_page); in migrate_vma_collect_pmd()
338 (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { in migrate_vma_collect_pmd()
340 migrate->cpages++; in migrate_vma_collect_pmd()
343 page = vm_normal_page(migrate->vma, addr, pte); in migrate_vma_collect_pmd()
345 !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { in migrate_vma_collect_pmd()
350 if (!(migrate->flags & in migrate_vma_collect_pmd()
352 pgmap->owner != migrate->pgmap_owner) in migrate_vma_collect_pmd()
362 migrate->fault_page); in migrate_vma_collect_pmd()
428 migrate->cpages++; in migrate_vma_collect_pmd()
480 migrate->dst[migrate->npages] = 0; in migrate_vma_collect_pmd()
481 migrate->src[migrate->npages++] = mpfn; in migrate_vma_collect_pmd()
508 static void migrate_vma_collect(struct migrate_vma *migrate) in migrate_vma_collect() argument
518 migrate->vma->vm_mm, migrate->start, migrate->end, in migrate_vma_collect()
519 migrate->pgmap_owner); in migrate_vma_collect()
522 walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end, in migrate_vma_collect()
523 &migrate_vma_walk_ops, migrate); in migrate_vma_collect()
526 migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT); in migrate_vma_collect()
665 static void migrate_vma_unmap(struct migrate_vma *migrate) in migrate_vma_unmap() argument
667 migrate->cpages = migrate_device_unmap(migrate->src, migrate->npages, in migrate_vma_unmap()
668 migrate->fault_page); in migrate_vma_unmap()
793 static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, in migrate_vma_insert_huge_pmd_page() argument
799 struct vm_area_struct *vma = migrate->vma; in migrate_vma_insert_huge_pmd_page()
823 if (mem_cgroup_charge(folio, migrate->vma->vm_mm, gfp)) { in migrate_vma_insert_huge_pmd_page()
908 static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, in migrate_vma_split_unmapped_folio() argument
918 split_huge_pmd_address(migrate->vma, addr, true); in migrate_vma_split_unmapped_folio()
922 migrate->src[idx] &= ~MIGRATE_PFN_COMPOUND; in migrate_vma_split_unmapped_folio()
923 flags = migrate->src[idx] & ((1UL << MIGRATE_PFN_SHIFT) - 1); in migrate_vma_split_unmapped_folio()
924 pfn = migrate->src[idx] >> MIGRATE_PFN_SHIFT; in migrate_vma_split_unmapped_folio()
926 migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; in migrate_vma_split_unmapped_folio()
930 static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, in migrate_vma_insert_huge_pmd_page() argument
939 static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, in migrate_vma_split_unmapped_folio() argument
968 static void migrate_vma_insert_page(struct migrate_vma *migrate, in migrate_vma_insert_page() argument
975 struct vm_area_struct *vma = migrate->vma; in migrate_vma_insert_page()
1003 int ret = migrate_vma_insert_huge_pmd_page(migrate, addr, page, in migrate_vma_insert_page()
1103 struct migrate_vma *migrate) in __migrate_device_pages() argument
1134 VM_BUG_ON(!migrate); in __migrate_device_pages()
1135 addr = migrate->start + i*PAGE_SIZE; in __migrate_device_pages()
1141 migrate->vma->vm_mm, addr, migrate->end, in __migrate_device_pages()
1142 migrate->pgmap_owner); in __migrate_device_pages()
1156 migrate_vma_insert_page(migrate, in __migrate_device_pages()
1176 if (!migrate) { in __migrate_device_pages()
1182 addr = migrate->start + i * PAGE_SIZE; in __migrate_device_pages()
1183 if (migrate_vma_split_unmapped_folio(migrate, i, addr, folio)) { in __migrate_device_pages()
1221 if (migrate && migrate->fault_page == page) in __migrate_device_pages()
1265 void migrate_vma_pages(struct migrate_vma *migrate) in migrate_vma_pages() argument
1267 __migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate); in migrate_vma_pages()
1351 void migrate_vma_finalize(struct migrate_vma *migrate) in migrate_vma_finalize() argument
1353 __migrate_device_finalize(migrate->src, migrate->dst, migrate->npages, in migrate_vma_finalize()
1354 migrate->fault_page); in migrate_vma_finalize()