11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Simple NUMA memory policy for the Linux kernel. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright 2003,2004 Andi Kleen, SuSE Labs. 58bccd85fSChristoph Lameter * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc. 61da177e4SLinus Torvalds * Subject to the GNU Public License, version 2. 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * NUMA policy allows the user to give hints in which node(s) memory should 91da177e4SLinus Torvalds * be allocated. 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * Support four policies per VMA and per process: 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * The VMA policy has priority over the process policy for a page fault. 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * interleave Allocate memory interleaved over a set of nodes, 161da177e4SLinus Torvalds * with normal fallback if it fails. 171da177e4SLinus Torvalds * For VMA based allocations this interleaves based on the 181da177e4SLinus Torvalds * offset into the backing object or offset into the mapping 191da177e4SLinus Torvalds * for anonymous memory. For process policy an process counter 201da177e4SLinus Torvalds * is used. 218bccd85fSChristoph Lameter * 221da177e4SLinus Torvalds * bind Only allocate memory on a specific set of nodes, 231da177e4SLinus Torvalds * no fallback. 248bccd85fSChristoph Lameter * FIXME: memory is allocated starting with the first node 258bccd85fSChristoph Lameter * to the last. It would be better if bind would truly restrict 268bccd85fSChristoph Lameter * the allocation to memory nodes instead 278bccd85fSChristoph Lameter * 281da177e4SLinus Torvalds * preferred Try a specific node first before normal fallback. 291da177e4SLinus Torvalds * As a special case node -1 here means do the allocation 301da177e4SLinus Torvalds * on the local CPU. This is normally identical to default, 311da177e4SLinus Torvalds * but useful to set in a VMA when you have a non default 321da177e4SLinus Torvalds * process policy. 338bccd85fSChristoph Lameter * 341da177e4SLinus Torvalds * default Allocate on the local node first, or when on a VMA 351da177e4SLinus Torvalds * use the process policy. This is what Linux always did 361da177e4SLinus Torvalds * in a NUMA aware kernel and still does by, ahem, default. 371da177e4SLinus Torvalds * 381da177e4SLinus Torvalds * The process policy is applied for most non interrupt memory allocations 391da177e4SLinus Torvalds * in that process' context. Interrupts ignore the policies and always 401da177e4SLinus Torvalds * try to allocate on the local CPU. The VMA policy is only applied for memory 411da177e4SLinus Torvalds * allocations for a VMA in the VM. 421da177e4SLinus Torvalds * 431da177e4SLinus Torvalds * Currently there are a few corner cases in swapping where the policy 441da177e4SLinus Torvalds * is not applied, but the majority should be handled. When process policy 451da177e4SLinus Torvalds * is used it is not remembered over swap outs/swap ins. 461da177e4SLinus Torvalds * 471da177e4SLinus Torvalds * Only the highest zone in the zone hierarchy gets policied. Allocations 481da177e4SLinus Torvalds * requesting a lower zone just use default policy. This implies that 491da177e4SLinus Torvalds * on systems with highmem kernel lowmem allocation don't get policied. 501da177e4SLinus Torvalds * Same with GFP_DMA allocations. 511da177e4SLinus Torvalds * 521da177e4SLinus Torvalds * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between 531da177e4SLinus Torvalds * all users and remembered even when nobody has memory mapped. 541da177e4SLinus Torvalds */ 551da177e4SLinus Torvalds 561da177e4SLinus Torvalds /* Notebook: 571da177e4SLinus Torvalds fix mmap readahead to honour policy and enable policy for any page cache 581da177e4SLinus Torvalds object 591da177e4SLinus Torvalds statistics for bigpages 601da177e4SLinus Torvalds global policy for page cache? currently it uses process policy. Requires 611da177e4SLinus Torvalds first item above. 621da177e4SLinus Torvalds handle mremap for shared memory (currently ignored for the policy) 631da177e4SLinus Torvalds grows down? 641da177e4SLinus Torvalds make bind policy root only? It can trigger oom much faster and the 651da177e4SLinus Torvalds kernel is not always grateful with that. 661da177e4SLinus Torvalds could replace all the switch()es with a mempolicy_ops structure. 671da177e4SLinus Torvalds */ 681da177e4SLinus Torvalds 691da177e4SLinus Torvalds #include <linux/mempolicy.h> 701da177e4SLinus Torvalds #include <linux/mm.h> 711da177e4SLinus Torvalds #include <linux/highmem.h> 721da177e4SLinus Torvalds #include <linux/hugetlb.h> 731da177e4SLinus Torvalds #include <linux/kernel.h> 741da177e4SLinus Torvalds #include <linux/sched.h> 751da177e4SLinus Torvalds #include <linux/mm.h> 761da177e4SLinus Torvalds #include <linux/nodemask.h> 771da177e4SLinus Torvalds #include <linux/cpuset.h> 781da177e4SLinus Torvalds #include <linux/gfp.h> 791da177e4SLinus Torvalds #include <linux/slab.h> 801da177e4SLinus Torvalds #include <linux/string.h> 811da177e4SLinus Torvalds #include <linux/module.h> 821da177e4SLinus Torvalds #include <linux/interrupt.h> 831da177e4SLinus Torvalds #include <linux/init.h> 841da177e4SLinus Torvalds #include <linux/compat.h> 851da177e4SLinus Torvalds #include <linux/mempolicy.h> 86dc9aa5b9SChristoph Lameter #include <linux/swap.h> 871a75a6c8SChristoph Lameter #include <linux/seq_file.h> 881a75a6c8SChristoph Lameter #include <linux/proc_fs.h> 89dc9aa5b9SChristoph Lameter 901da177e4SLinus Torvalds #include <asm/tlbflush.h> 911da177e4SLinus Torvalds #include <asm/uaccess.h> 921da177e4SLinus Torvalds 9338e35860SChristoph Lameter /* Internal flags */ 94dc9aa5b9SChristoph Lameter #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ 9538e35860SChristoph Lameter #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */ 961a75a6c8SChristoph Lameter #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */ 97dc9aa5b9SChristoph Lameter 987e2ab150SChristoph Lameter /* The number of pages to migrate per call to migrate_pages() */ 997e2ab150SChristoph Lameter #define MIGRATE_CHUNK_SIZE 256 1007e2ab150SChristoph Lameter 1011da177e4SLinus Torvalds static kmem_cache_t *policy_cache; 1021da177e4SLinus Torvalds static kmem_cache_t *sn_cache; 1031da177e4SLinus Torvalds 1041da177e4SLinus Torvalds #define PDprintk(fmt...) 1051da177e4SLinus Torvalds 1061da177e4SLinus Torvalds /* Highest zone. An specific allocation for a zone below that is not 1071da177e4SLinus Torvalds policied. */ 1084be38e35SChristoph Lameter int policy_zone = ZONE_DMA; 1091da177e4SLinus Torvalds 110d42c6997SAndi Kleen struct mempolicy default_policy = { 1111da177e4SLinus Torvalds .refcnt = ATOMIC_INIT(1), /* never free it */ 1121da177e4SLinus Torvalds .policy = MPOL_DEFAULT, 1131da177e4SLinus Torvalds }; 1141da177e4SLinus Torvalds 1151da177e4SLinus Torvalds /* Do sanity checking on a policy */ 116dfcd3c0dSAndi Kleen static int mpol_check_policy(int mode, nodemask_t *nodes) 1171da177e4SLinus Torvalds { 118dfcd3c0dSAndi Kleen int empty = nodes_empty(*nodes); 1191da177e4SLinus Torvalds 1201da177e4SLinus Torvalds switch (mode) { 1211da177e4SLinus Torvalds case MPOL_DEFAULT: 1221da177e4SLinus Torvalds if (!empty) 1231da177e4SLinus Torvalds return -EINVAL; 1241da177e4SLinus Torvalds break; 1251da177e4SLinus Torvalds case MPOL_BIND: 1261da177e4SLinus Torvalds case MPOL_INTERLEAVE: 1271da177e4SLinus Torvalds /* Preferred will only use the first bit, but allow 1281da177e4SLinus Torvalds more for now. */ 1291da177e4SLinus Torvalds if (empty) 1301da177e4SLinus Torvalds return -EINVAL; 1311da177e4SLinus Torvalds break; 1321da177e4SLinus Torvalds } 133dfcd3c0dSAndi Kleen return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL; 1341da177e4SLinus Torvalds } 1351da177e4SLinus Torvalds /* Generate a custom zonelist for the BIND policy. */ 136dfcd3c0dSAndi Kleen static struct zonelist *bind_zonelist(nodemask_t *nodes) 1371da177e4SLinus Torvalds { 1381da177e4SLinus Torvalds struct zonelist *zl; 1391da177e4SLinus Torvalds int num, max, nd; 1401da177e4SLinus Torvalds 141dfcd3c0dSAndi Kleen max = 1 + MAX_NR_ZONES * nodes_weight(*nodes); 1421da177e4SLinus Torvalds zl = kmalloc(sizeof(void *) * max, GFP_KERNEL); 1431da177e4SLinus Torvalds if (!zl) 1441da177e4SLinus Torvalds return NULL; 1451da177e4SLinus Torvalds num = 0; 1464be38e35SChristoph Lameter for_each_node_mask(nd, *nodes) 1474be38e35SChristoph Lameter zl->zones[num++] = &NODE_DATA(nd)->node_zones[policy_zone]; 1481da177e4SLinus Torvalds zl->zones[num] = NULL; 1491da177e4SLinus Torvalds return zl; 1501da177e4SLinus Torvalds } 1511da177e4SLinus Torvalds 1521da177e4SLinus Torvalds /* Create a new policy */ 153dfcd3c0dSAndi Kleen static struct mempolicy *mpol_new(int mode, nodemask_t *nodes) 1541da177e4SLinus Torvalds { 1551da177e4SLinus Torvalds struct mempolicy *policy; 1561da177e4SLinus Torvalds 157dfcd3c0dSAndi Kleen PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes_addr(*nodes)[0]); 1581da177e4SLinus Torvalds if (mode == MPOL_DEFAULT) 1591da177e4SLinus Torvalds return NULL; 1601da177e4SLinus Torvalds policy = kmem_cache_alloc(policy_cache, GFP_KERNEL); 1611da177e4SLinus Torvalds if (!policy) 1621da177e4SLinus Torvalds return ERR_PTR(-ENOMEM); 1631da177e4SLinus Torvalds atomic_set(&policy->refcnt, 1); 1641da177e4SLinus Torvalds switch (mode) { 1651da177e4SLinus Torvalds case MPOL_INTERLEAVE: 166dfcd3c0dSAndi Kleen policy->v.nodes = *nodes; 1678f493d79SAndi Kleen if (nodes_weight(*nodes) == 0) { 1688f493d79SAndi Kleen kmem_cache_free(policy_cache, policy); 1698f493d79SAndi Kleen return ERR_PTR(-EINVAL); 1708f493d79SAndi Kleen } 1711da177e4SLinus Torvalds break; 1721da177e4SLinus Torvalds case MPOL_PREFERRED: 173dfcd3c0dSAndi Kleen policy->v.preferred_node = first_node(*nodes); 1741da177e4SLinus Torvalds if (policy->v.preferred_node >= MAX_NUMNODES) 1751da177e4SLinus Torvalds policy->v.preferred_node = -1; 1761da177e4SLinus Torvalds break; 1771da177e4SLinus Torvalds case MPOL_BIND: 1781da177e4SLinus Torvalds policy->v.zonelist = bind_zonelist(nodes); 1791da177e4SLinus Torvalds if (policy->v.zonelist == NULL) { 1801da177e4SLinus Torvalds kmem_cache_free(policy_cache, policy); 1811da177e4SLinus Torvalds return ERR_PTR(-ENOMEM); 1821da177e4SLinus Torvalds } 1831da177e4SLinus Torvalds break; 1841da177e4SLinus Torvalds } 1851da177e4SLinus Torvalds policy->policy = mode; 18674cb2155SPaul Jackson policy->cpuset_mems_allowed = cpuset_mems_allowed(current); 1871da177e4SLinus Torvalds return policy; 1881da177e4SLinus Torvalds } 1891da177e4SLinus Torvalds 1901a75a6c8SChristoph Lameter static void gather_stats(struct page *, void *); 191fc301289SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist, 192fc301289SChristoph Lameter unsigned long flags); 1931a75a6c8SChristoph Lameter 19438e35860SChristoph Lameter /* Scan through pages checking if pages follow certain conditions. */ 195b5810039SNick Piggin static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 196dc9aa5b9SChristoph Lameter unsigned long addr, unsigned long end, 197dc9aa5b9SChristoph Lameter const nodemask_t *nodes, unsigned long flags, 19838e35860SChristoph Lameter void *private) 1991da177e4SLinus Torvalds { 20091612e0dSHugh Dickins pte_t *orig_pte; 20191612e0dSHugh Dickins pte_t *pte; 202705e87c0SHugh Dickins spinlock_t *ptl; 203941150a3SHugh Dickins 204705e87c0SHugh Dickins orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 20591612e0dSHugh Dickins do { 2066aab341eSLinus Torvalds struct page *page; 20791612e0dSHugh Dickins unsigned int nid; 20891612e0dSHugh Dickins 20991612e0dSHugh Dickins if (!pte_present(*pte)) 21091612e0dSHugh Dickins continue; 2116aab341eSLinus Torvalds page = vm_normal_page(vma, addr, *pte); 2126aab341eSLinus Torvalds if (!page) 21391612e0dSHugh Dickins continue; 214053837fcSNick Piggin /* 215053837fcSNick Piggin * The check for PageReserved here is important to avoid 216053837fcSNick Piggin * handling zero pages and other pages that may have been 217053837fcSNick Piggin * marked special by the system. 218053837fcSNick Piggin * 219053837fcSNick Piggin * If the PageReserved would not be checked here then f.e. 220053837fcSNick Piggin * the location of the zero page could have an influence 221053837fcSNick Piggin * on MPOL_MF_STRICT, zero pages would be counted for 222053837fcSNick Piggin * the per node stats, and there would be useless attempts 223053837fcSNick Piggin * to put zero pages on the migration list. 224053837fcSNick Piggin */ 225f4598c8bSChristoph Lameter if (PageReserved(page)) 226f4598c8bSChristoph Lameter continue; 2276aab341eSLinus Torvalds nid = page_to_nid(page); 22838e35860SChristoph Lameter if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT)) 22938e35860SChristoph Lameter continue; 23038e35860SChristoph Lameter 2311a75a6c8SChristoph Lameter if (flags & MPOL_MF_STATS) 2321a75a6c8SChristoph Lameter gather_stats(page, private); 233053837fcSNick Piggin else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) 234fc301289SChristoph Lameter migrate_page_add(page, private, flags); 235dc9aa5b9SChristoph Lameter else 2361da177e4SLinus Torvalds break; 23791612e0dSHugh Dickins } while (pte++, addr += PAGE_SIZE, addr != end); 238705e87c0SHugh Dickins pte_unmap_unlock(orig_pte, ptl); 23991612e0dSHugh Dickins return addr != end; 24091612e0dSHugh Dickins } 24191612e0dSHugh Dickins 242b5810039SNick Piggin static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud, 243dc9aa5b9SChristoph Lameter unsigned long addr, unsigned long end, 244dc9aa5b9SChristoph Lameter const nodemask_t *nodes, unsigned long flags, 24538e35860SChristoph Lameter void *private) 24691612e0dSHugh Dickins { 24791612e0dSHugh Dickins pmd_t *pmd; 24891612e0dSHugh Dickins unsigned long next; 24991612e0dSHugh Dickins 25091612e0dSHugh Dickins pmd = pmd_offset(pud, addr); 25191612e0dSHugh Dickins do { 25291612e0dSHugh Dickins next = pmd_addr_end(addr, end); 25391612e0dSHugh Dickins if (pmd_none_or_clear_bad(pmd)) 25491612e0dSHugh Dickins continue; 255dc9aa5b9SChristoph Lameter if (check_pte_range(vma, pmd, addr, next, nodes, 25638e35860SChristoph Lameter flags, private)) 25791612e0dSHugh Dickins return -EIO; 25891612e0dSHugh Dickins } while (pmd++, addr = next, addr != end); 25991612e0dSHugh Dickins return 0; 26091612e0dSHugh Dickins } 26191612e0dSHugh Dickins 262b5810039SNick Piggin static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd, 263dc9aa5b9SChristoph Lameter unsigned long addr, unsigned long end, 264dc9aa5b9SChristoph Lameter const nodemask_t *nodes, unsigned long flags, 26538e35860SChristoph Lameter void *private) 26691612e0dSHugh Dickins { 26791612e0dSHugh Dickins pud_t *pud; 26891612e0dSHugh Dickins unsigned long next; 26991612e0dSHugh Dickins 27091612e0dSHugh Dickins pud = pud_offset(pgd, addr); 27191612e0dSHugh Dickins do { 27291612e0dSHugh Dickins next = pud_addr_end(addr, end); 27391612e0dSHugh Dickins if (pud_none_or_clear_bad(pud)) 27491612e0dSHugh Dickins continue; 275dc9aa5b9SChristoph Lameter if (check_pmd_range(vma, pud, addr, next, nodes, 27638e35860SChristoph Lameter flags, private)) 27791612e0dSHugh Dickins return -EIO; 27891612e0dSHugh Dickins } while (pud++, addr = next, addr != end); 27991612e0dSHugh Dickins return 0; 28091612e0dSHugh Dickins } 28191612e0dSHugh Dickins 282b5810039SNick Piggin static inline int check_pgd_range(struct vm_area_struct *vma, 283dc9aa5b9SChristoph Lameter unsigned long addr, unsigned long end, 284dc9aa5b9SChristoph Lameter const nodemask_t *nodes, unsigned long flags, 28538e35860SChristoph Lameter void *private) 28691612e0dSHugh Dickins { 28791612e0dSHugh Dickins pgd_t *pgd; 28891612e0dSHugh Dickins unsigned long next; 28991612e0dSHugh Dickins 290b5810039SNick Piggin pgd = pgd_offset(vma->vm_mm, addr); 29191612e0dSHugh Dickins do { 29291612e0dSHugh Dickins next = pgd_addr_end(addr, end); 29391612e0dSHugh Dickins if (pgd_none_or_clear_bad(pgd)) 29491612e0dSHugh Dickins continue; 295dc9aa5b9SChristoph Lameter if (check_pud_range(vma, pgd, addr, next, nodes, 29638e35860SChristoph Lameter flags, private)) 29791612e0dSHugh Dickins return -EIO; 29891612e0dSHugh Dickins } while (pgd++, addr = next, addr != end); 29991612e0dSHugh Dickins return 0; 3001da177e4SLinus Torvalds } 3011da177e4SLinus Torvalds 302dc9aa5b9SChristoph Lameter /* Check if a vma is migratable */ 303dc9aa5b9SChristoph Lameter static inline int vma_migratable(struct vm_area_struct *vma) 304dc9aa5b9SChristoph Lameter { 305dc9aa5b9SChristoph Lameter if (vma->vm_flags & ( 306f4598c8bSChristoph Lameter VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED)) 307dc9aa5b9SChristoph Lameter return 0; 308dc9aa5b9SChristoph Lameter return 1; 309dc9aa5b9SChristoph Lameter } 310dc9aa5b9SChristoph Lameter 311dc9aa5b9SChristoph Lameter /* 312dc9aa5b9SChristoph Lameter * Check if all pages in a range are on a set of nodes. 313dc9aa5b9SChristoph Lameter * If pagelist != NULL then isolate pages from the LRU and 314dc9aa5b9SChristoph Lameter * put them on the pagelist. 315dc9aa5b9SChristoph Lameter */ 3161da177e4SLinus Torvalds static struct vm_area_struct * 3171da177e4SLinus Torvalds check_range(struct mm_struct *mm, unsigned long start, unsigned long end, 31838e35860SChristoph Lameter const nodemask_t *nodes, unsigned long flags, void *private) 3191da177e4SLinus Torvalds { 3201da177e4SLinus Torvalds int err; 3211da177e4SLinus Torvalds struct vm_area_struct *first, *vma, *prev; 3221da177e4SLinus Torvalds 323053837fcSNick Piggin /* Clear the LRU lists so pages can be isolated */ 324053837fcSNick Piggin if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) 325053837fcSNick Piggin lru_add_drain_all(); 326053837fcSNick Piggin 3271da177e4SLinus Torvalds first = find_vma(mm, start); 3281da177e4SLinus Torvalds if (!first) 3291da177e4SLinus Torvalds return ERR_PTR(-EFAULT); 3301da177e4SLinus Torvalds prev = NULL; 3311da177e4SLinus Torvalds for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) { 332dc9aa5b9SChristoph Lameter if (!(flags & MPOL_MF_DISCONTIG_OK)) { 3331da177e4SLinus Torvalds if (!vma->vm_next && vma->vm_end < end) 3341da177e4SLinus Torvalds return ERR_PTR(-EFAULT); 3351da177e4SLinus Torvalds if (prev && prev->vm_end < vma->vm_start) 3361da177e4SLinus Torvalds return ERR_PTR(-EFAULT); 337dc9aa5b9SChristoph Lameter } 338dc9aa5b9SChristoph Lameter if (!is_vm_hugetlb_page(vma) && 339dc9aa5b9SChristoph Lameter ((flags & MPOL_MF_STRICT) || 340dc9aa5b9SChristoph Lameter ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) && 341dc9aa5b9SChristoph Lameter vma_migratable(vma)))) { 3425b952b3cSAndi Kleen unsigned long endvma = vma->vm_end; 343dc9aa5b9SChristoph Lameter 3445b952b3cSAndi Kleen if (endvma > end) 3455b952b3cSAndi Kleen endvma = end; 3465b952b3cSAndi Kleen if (vma->vm_start > start) 3475b952b3cSAndi Kleen start = vma->vm_start; 348dc9aa5b9SChristoph Lameter err = check_pgd_range(vma, start, endvma, nodes, 34938e35860SChristoph Lameter flags, private); 3501da177e4SLinus Torvalds if (err) { 3511da177e4SLinus Torvalds first = ERR_PTR(err); 3521da177e4SLinus Torvalds break; 3531da177e4SLinus Torvalds } 3541da177e4SLinus Torvalds } 3551da177e4SLinus Torvalds prev = vma; 3561da177e4SLinus Torvalds } 3571da177e4SLinus Torvalds return first; 3581da177e4SLinus Torvalds } 3591da177e4SLinus Torvalds 3601da177e4SLinus Torvalds /* Apply policy to a single VMA */ 3611da177e4SLinus Torvalds static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new) 3621da177e4SLinus Torvalds { 3631da177e4SLinus Torvalds int err = 0; 3641da177e4SLinus Torvalds struct mempolicy *old = vma->vm_policy; 3651da177e4SLinus Torvalds 3661da177e4SLinus Torvalds PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n", 3671da177e4SLinus Torvalds vma->vm_start, vma->vm_end, vma->vm_pgoff, 3681da177e4SLinus Torvalds vma->vm_ops, vma->vm_file, 3691da177e4SLinus Torvalds vma->vm_ops ? vma->vm_ops->set_policy : NULL); 3701da177e4SLinus Torvalds 3711da177e4SLinus Torvalds if (vma->vm_ops && vma->vm_ops->set_policy) 3721da177e4SLinus Torvalds err = vma->vm_ops->set_policy(vma, new); 3731da177e4SLinus Torvalds if (!err) { 3741da177e4SLinus Torvalds mpol_get(new); 3751da177e4SLinus Torvalds vma->vm_policy = new; 3761da177e4SLinus Torvalds mpol_free(old); 3771da177e4SLinus Torvalds } 3781da177e4SLinus Torvalds return err; 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds 3811da177e4SLinus Torvalds /* Step 2: apply policy to a range and do splits. */ 3821da177e4SLinus Torvalds static int mbind_range(struct vm_area_struct *vma, unsigned long start, 3831da177e4SLinus Torvalds unsigned long end, struct mempolicy *new) 3841da177e4SLinus Torvalds { 3851da177e4SLinus Torvalds struct vm_area_struct *next; 3861da177e4SLinus Torvalds int err; 3871da177e4SLinus Torvalds 3881da177e4SLinus Torvalds err = 0; 3891da177e4SLinus Torvalds for (; vma && vma->vm_start < end; vma = next) { 3901da177e4SLinus Torvalds next = vma->vm_next; 3911da177e4SLinus Torvalds if (vma->vm_start < start) 3921da177e4SLinus Torvalds err = split_vma(vma->vm_mm, vma, start, 1); 3931da177e4SLinus Torvalds if (!err && vma->vm_end > end) 3941da177e4SLinus Torvalds err = split_vma(vma->vm_mm, vma, end, 0); 3951da177e4SLinus Torvalds if (!err) 3961da177e4SLinus Torvalds err = policy_vma(vma, new); 3971da177e4SLinus Torvalds if (err) 3981da177e4SLinus Torvalds break; 3991da177e4SLinus Torvalds } 4001da177e4SLinus Torvalds return err; 4011da177e4SLinus Torvalds } 4021da177e4SLinus Torvalds 4038bccd85fSChristoph Lameter static int contextualize_policy(int mode, nodemask_t *nodes) 4048bccd85fSChristoph Lameter { 4058bccd85fSChristoph Lameter if (!nodes) 4068bccd85fSChristoph Lameter return 0; 4078bccd85fSChristoph Lameter 408cf2a473cSPaul Jackson cpuset_update_task_memory_state(); 4095966514dSPaul Jackson if (!cpuset_nodes_subset_current_mems_allowed(*nodes)) 4105966514dSPaul Jackson return -EINVAL; 4118bccd85fSChristoph Lameter return mpol_check_policy(mode, nodes); 4128bccd85fSChristoph Lameter } 4138bccd85fSChristoph Lameter 4141da177e4SLinus Torvalds /* Set the process memory policy */ 4158bccd85fSChristoph Lameter long do_set_mempolicy(int mode, nodemask_t *nodes) 4161da177e4SLinus Torvalds { 4171da177e4SLinus Torvalds struct mempolicy *new; 4181da177e4SLinus Torvalds 4198bccd85fSChristoph Lameter if (contextualize_policy(mode, nodes)) 4201da177e4SLinus Torvalds return -EINVAL; 4218bccd85fSChristoph Lameter new = mpol_new(mode, nodes); 4221da177e4SLinus Torvalds if (IS_ERR(new)) 4231da177e4SLinus Torvalds return PTR_ERR(new); 4241da177e4SLinus Torvalds mpol_free(current->mempolicy); 4251da177e4SLinus Torvalds current->mempolicy = new; 4261da177e4SLinus Torvalds if (new && new->policy == MPOL_INTERLEAVE) 427dfcd3c0dSAndi Kleen current->il_next = first_node(new->v.nodes); 4281da177e4SLinus Torvalds return 0; 4291da177e4SLinus Torvalds } 4301da177e4SLinus Torvalds 4311da177e4SLinus Torvalds /* Fill a zone bitmap for a policy */ 432dfcd3c0dSAndi Kleen static void get_zonemask(struct mempolicy *p, nodemask_t *nodes) 4331da177e4SLinus Torvalds { 4341da177e4SLinus Torvalds int i; 4351da177e4SLinus Torvalds 436dfcd3c0dSAndi Kleen nodes_clear(*nodes); 4371da177e4SLinus Torvalds switch (p->policy) { 4381da177e4SLinus Torvalds case MPOL_BIND: 4391da177e4SLinus Torvalds for (i = 0; p->v.zonelist->zones[i]; i++) 4408bccd85fSChristoph Lameter node_set(p->v.zonelist->zones[i]->zone_pgdat->node_id, 4418bccd85fSChristoph Lameter *nodes); 4421da177e4SLinus Torvalds break; 4431da177e4SLinus Torvalds case MPOL_DEFAULT: 4441da177e4SLinus Torvalds break; 4451da177e4SLinus Torvalds case MPOL_INTERLEAVE: 446dfcd3c0dSAndi Kleen *nodes = p->v.nodes; 4471da177e4SLinus Torvalds break; 4481da177e4SLinus Torvalds case MPOL_PREFERRED: 4491da177e4SLinus Torvalds /* or use current node instead of online map? */ 4501da177e4SLinus Torvalds if (p->v.preferred_node < 0) 451dfcd3c0dSAndi Kleen *nodes = node_online_map; 4521da177e4SLinus Torvalds else 453dfcd3c0dSAndi Kleen node_set(p->v.preferred_node, *nodes); 4541da177e4SLinus Torvalds break; 4551da177e4SLinus Torvalds default: 4561da177e4SLinus Torvalds BUG(); 4571da177e4SLinus Torvalds } 4581da177e4SLinus Torvalds } 4591da177e4SLinus Torvalds 4601da177e4SLinus Torvalds static int lookup_node(struct mm_struct *mm, unsigned long addr) 4611da177e4SLinus Torvalds { 4621da177e4SLinus Torvalds struct page *p; 4631da177e4SLinus Torvalds int err; 4641da177e4SLinus Torvalds 4651da177e4SLinus Torvalds err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL); 4661da177e4SLinus Torvalds if (err >= 0) { 4671da177e4SLinus Torvalds err = page_to_nid(p); 4681da177e4SLinus Torvalds put_page(p); 4691da177e4SLinus Torvalds } 4701da177e4SLinus Torvalds return err; 4711da177e4SLinus Torvalds } 4721da177e4SLinus Torvalds 4731da177e4SLinus Torvalds /* Retrieve NUMA policy */ 4748bccd85fSChristoph Lameter long do_get_mempolicy(int *policy, nodemask_t *nmask, 4751da177e4SLinus Torvalds unsigned long addr, unsigned long flags) 4761da177e4SLinus Torvalds { 4778bccd85fSChristoph Lameter int err; 4781da177e4SLinus Torvalds struct mm_struct *mm = current->mm; 4791da177e4SLinus Torvalds struct vm_area_struct *vma = NULL; 4801da177e4SLinus Torvalds struct mempolicy *pol = current->mempolicy; 4811da177e4SLinus Torvalds 482cf2a473cSPaul Jackson cpuset_update_task_memory_state(); 4831da177e4SLinus Torvalds if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR)) 4841da177e4SLinus Torvalds return -EINVAL; 4851da177e4SLinus Torvalds if (flags & MPOL_F_ADDR) { 4861da177e4SLinus Torvalds down_read(&mm->mmap_sem); 4871da177e4SLinus Torvalds vma = find_vma_intersection(mm, addr, addr+1); 4881da177e4SLinus Torvalds if (!vma) { 4891da177e4SLinus Torvalds up_read(&mm->mmap_sem); 4901da177e4SLinus Torvalds return -EFAULT; 4911da177e4SLinus Torvalds } 4921da177e4SLinus Torvalds if (vma->vm_ops && vma->vm_ops->get_policy) 4931da177e4SLinus Torvalds pol = vma->vm_ops->get_policy(vma, addr); 4941da177e4SLinus Torvalds else 4951da177e4SLinus Torvalds pol = vma->vm_policy; 4961da177e4SLinus Torvalds } else if (addr) 4971da177e4SLinus Torvalds return -EINVAL; 4981da177e4SLinus Torvalds 4991da177e4SLinus Torvalds if (!pol) 5001da177e4SLinus Torvalds pol = &default_policy; 5011da177e4SLinus Torvalds 5021da177e4SLinus Torvalds if (flags & MPOL_F_NODE) { 5031da177e4SLinus Torvalds if (flags & MPOL_F_ADDR) { 5041da177e4SLinus Torvalds err = lookup_node(mm, addr); 5051da177e4SLinus Torvalds if (err < 0) 5061da177e4SLinus Torvalds goto out; 5078bccd85fSChristoph Lameter *policy = err; 5081da177e4SLinus Torvalds } else if (pol == current->mempolicy && 5091da177e4SLinus Torvalds pol->policy == MPOL_INTERLEAVE) { 5108bccd85fSChristoph Lameter *policy = current->il_next; 5111da177e4SLinus Torvalds } else { 5121da177e4SLinus Torvalds err = -EINVAL; 5131da177e4SLinus Torvalds goto out; 5141da177e4SLinus Torvalds } 5151da177e4SLinus Torvalds } else 5168bccd85fSChristoph Lameter *policy = pol->policy; 5171da177e4SLinus Torvalds 5181da177e4SLinus Torvalds if (vma) { 5191da177e4SLinus Torvalds up_read(¤t->mm->mmap_sem); 5201da177e4SLinus Torvalds vma = NULL; 5211da177e4SLinus Torvalds } 5221da177e4SLinus Torvalds 5231da177e4SLinus Torvalds err = 0; 5248bccd85fSChristoph Lameter if (nmask) 5258bccd85fSChristoph Lameter get_zonemask(pol, nmask); 5261da177e4SLinus Torvalds 5271da177e4SLinus Torvalds out: 5281da177e4SLinus Torvalds if (vma) 5291da177e4SLinus Torvalds up_read(¤t->mm->mmap_sem); 5301da177e4SLinus Torvalds return err; 5311da177e4SLinus Torvalds } 5321da177e4SLinus Torvalds 5338bccd85fSChristoph Lameter /* 5346ce3c4c0SChristoph Lameter * page migration 5356ce3c4c0SChristoph Lameter */ 5366ce3c4c0SChristoph Lameter 537fc301289SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist, 538fc301289SChristoph Lameter unsigned long flags) 5396ce3c4c0SChristoph Lameter { 5406ce3c4c0SChristoph Lameter /* 541fc301289SChristoph Lameter * Avoid migrating a page that is shared with others. 5426ce3c4c0SChristoph Lameter */ 543fc301289SChristoph Lameter if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) { 544053837fcSNick Piggin if (isolate_lru_page(page)) 5456ce3c4c0SChristoph Lameter list_add(&page->lru, pagelist); 5466ce3c4c0SChristoph Lameter } 5476ce3c4c0SChristoph Lameter } 5486ce3c4c0SChristoph Lameter 5497e2ab150SChristoph Lameter /* 5507e2ab150SChristoph Lameter * Migrate the list 'pagelist' of pages to a certain destination. 5517e2ab150SChristoph Lameter * 5527e2ab150SChristoph Lameter * Specify destination with either non-NULL vma or dest_node >= 0 5537e2ab150SChristoph Lameter * Return the number of pages not migrated or error code 5547e2ab150SChristoph Lameter */ 5557e2ab150SChristoph Lameter static int migrate_pages_to(struct list_head *pagelist, 5567e2ab150SChristoph Lameter struct vm_area_struct *vma, int dest) 5576ce3c4c0SChristoph Lameter { 5587e2ab150SChristoph Lameter LIST_HEAD(newlist); 5596ce3c4c0SChristoph Lameter LIST_HEAD(moved); 5606ce3c4c0SChristoph Lameter LIST_HEAD(failed); 5617e2ab150SChristoph Lameter int err = 0; 5627e2ab150SChristoph Lameter int nr_pages; 5637e2ab150SChristoph Lameter struct page *page; 5647e2ab150SChristoph Lameter struct list_head *p; 5656ce3c4c0SChristoph Lameter 5667e2ab150SChristoph Lameter redo: 5677e2ab150SChristoph Lameter nr_pages = 0; 5687e2ab150SChristoph Lameter list_for_each(p, pagelist) { 5697e2ab150SChristoph Lameter if (vma) 5707e2ab150SChristoph Lameter page = alloc_page_vma(GFP_HIGHUSER, vma, vma->vm_start); 5717e2ab150SChristoph Lameter else 5727e2ab150SChristoph Lameter page = alloc_pages_node(dest, GFP_HIGHUSER, 0); 5736ce3c4c0SChristoph Lameter 5747e2ab150SChristoph Lameter if (!page) { 5757e2ab150SChristoph Lameter err = -ENOMEM; 5767e2ab150SChristoph Lameter goto out; 5777e2ab150SChristoph Lameter } 5787e2ab150SChristoph Lameter list_add(&page->lru, &newlist); 5797e2ab150SChristoph Lameter nr_pages++; 5807e2ab150SChristoph Lameter if (nr_pages > MIGRATE_CHUNK_SIZE); 5817e2ab150SChristoph Lameter break; 5827e2ab150SChristoph Lameter } 5837e2ab150SChristoph Lameter err = migrate_pages(pagelist, &newlist, &moved, &failed); 5847e2ab150SChristoph Lameter 5857e2ab150SChristoph Lameter putback_lru_pages(&moved); /* Call release pages instead ?? */ 5867e2ab150SChristoph Lameter 5877e2ab150SChristoph Lameter if (err >= 0 && list_empty(&newlist) && !list_empty(pagelist)) 5887e2ab150SChristoph Lameter goto redo; 5897e2ab150SChristoph Lameter out: 5907e2ab150SChristoph Lameter /* Return leftover allocated pages */ 5917e2ab150SChristoph Lameter while (!list_empty(&newlist)) { 5927e2ab150SChristoph Lameter page = list_entry(newlist.next, struct page, lru); 5937e2ab150SChristoph Lameter list_del(&page->lru); 5947e2ab150SChristoph Lameter __free_page(page); 5957e2ab150SChristoph Lameter } 5967e2ab150SChristoph Lameter list_splice(&failed, pagelist); 5977e2ab150SChristoph Lameter if (err < 0) 5987e2ab150SChristoph Lameter return err; 5997e2ab150SChristoph Lameter 6007e2ab150SChristoph Lameter /* Calculate number of leftover pages */ 6017e2ab150SChristoph Lameter nr_pages = 0; 6027e2ab150SChristoph Lameter list_for_each(p, pagelist) 6037e2ab150SChristoph Lameter nr_pages++; 6047e2ab150SChristoph Lameter return nr_pages; 6056ce3c4c0SChristoph Lameter } 6066ce3c4c0SChristoph Lameter 6076ce3c4c0SChristoph Lameter /* 6087e2ab150SChristoph Lameter * Migrate pages from one node to a target node. 6097e2ab150SChristoph Lameter * Returns error or the number of pages not migrated. 6107e2ab150SChristoph Lameter */ 6117e2ab150SChristoph Lameter int migrate_to_node(struct mm_struct *mm, int source, int dest, int flags) 6127e2ab150SChristoph Lameter { 6137e2ab150SChristoph Lameter nodemask_t nmask; 6147e2ab150SChristoph Lameter LIST_HEAD(pagelist); 6157e2ab150SChristoph Lameter int err = 0; 6167e2ab150SChristoph Lameter 6177e2ab150SChristoph Lameter nodes_clear(nmask); 6187e2ab150SChristoph Lameter node_set(source, nmask); 6197e2ab150SChristoph Lameter 6207e2ab150SChristoph Lameter check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask, 6217e2ab150SChristoph Lameter flags | MPOL_MF_DISCONTIG_OK, &pagelist); 6227e2ab150SChristoph Lameter 6237e2ab150SChristoph Lameter if (!list_empty(&pagelist)) { 6247e2ab150SChristoph Lameter err = migrate_pages_to(&pagelist, NULL, dest); 6257e2ab150SChristoph Lameter if (!list_empty(&pagelist)) 6267e2ab150SChristoph Lameter putback_lru_pages(&pagelist); 6277e2ab150SChristoph Lameter } 6287e2ab150SChristoph Lameter return err; 6297e2ab150SChristoph Lameter } 6307e2ab150SChristoph Lameter 6317e2ab150SChristoph Lameter /* 6327e2ab150SChristoph Lameter * Move pages between the two nodesets so as to preserve the physical 6337e2ab150SChristoph Lameter * layout as much as possible. 63439743889SChristoph Lameter * 63539743889SChristoph Lameter * Returns the number of page that could not be moved. 63639743889SChristoph Lameter */ 63739743889SChristoph Lameter int do_migrate_pages(struct mm_struct *mm, 63839743889SChristoph Lameter const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags) 63939743889SChristoph Lameter { 64039743889SChristoph Lameter LIST_HEAD(pagelist); 6417e2ab150SChristoph Lameter int busy = 0; 6427e2ab150SChristoph Lameter int err = 0; 6437e2ab150SChristoph Lameter nodemask_t tmp; 64439743889SChristoph Lameter 64539743889SChristoph Lameter down_read(&mm->mmap_sem); 646d4984711SChristoph Lameter 6477e2ab150SChristoph Lameter /* 6487e2ab150SChristoph Lameter * Find a 'source' bit set in 'tmp' whose corresponding 'dest' 6497e2ab150SChristoph Lameter * bit in 'to' is not also set in 'tmp'. Clear the found 'source' 6507e2ab150SChristoph Lameter * bit in 'tmp', and return that <source, dest> pair for migration. 6517e2ab150SChristoph Lameter * The pair of nodemasks 'to' and 'from' define the map. 6527e2ab150SChristoph Lameter * 6537e2ab150SChristoph Lameter * If no pair of bits is found that way, fallback to picking some 6547e2ab150SChristoph Lameter * pair of 'source' and 'dest' bits that are not the same. If the 6557e2ab150SChristoph Lameter * 'source' and 'dest' bits are the same, this represents a node 6567e2ab150SChristoph Lameter * that will be migrating to itself, so no pages need move. 6577e2ab150SChristoph Lameter * 6587e2ab150SChristoph Lameter * If no bits are left in 'tmp', or if all remaining bits left 6597e2ab150SChristoph Lameter * in 'tmp' correspond to the same bit in 'to', return false 6607e2ab150SChristoph Lameter * (nothing left to migrate). 6617e2ab150SChristoph Lameter * 6627e2ab150SChristoph Lameter * This lets us pick a pair of nodes to migrate between, such that 6637e2ab150SChristoph Lameter * if possible the dest node is not already occupied by some other 6647e2ab150SChristoph Lameter * source node, minimizing the risk of overloading the memory on a 6657e2ab150SChristoph Lameter * node that would happen if we migrated incoming memory to a node 6667e2ab150SChristoph Lameter * before migrating outgoing memory source that same node. 6677e2ab150SChristoph Lameter * 6687e2ab150SChristoph Lameter * A single scan of tmp is sufficient. As we go, we remember the 6697e2ab150SChristoph Lameter * most recent <s, d> pair that moved (s != d). If we find a pair 6707e2ab150SChristoph Lameter * that not only moved, but what's better, moved to an empty slot 6717e2ab150SChristoph Lameter * (d is not set in tmp), then we break out then, with that pair. 6727e2ab150SChristoph Lameter * Otherwise when we finish scannng from_tmp, we at least have the 6737e2ab150SChristoph Lameter * most recent <s, d> pair that moved. If we get all the way through 6747e2ab150SChristoph Lameter * the scan of tmp without finding any node that moved, much less 6757e2ab150SChristoph Lameter * moved to an empty node, then there is nothing left worth migrating. 6767e2ab150SChristoph Lameter */ 6777e2ab150SChristoph Lameter 6787e2ab150SChristoph Lameter tmp = *from_nodes; 6797e2ab150SChristoph Lameter while (!nodes_empty(tmp)) { 6807e2ab150SChristoph Lameter int s,d; 6817e2ab150SChristoph Lameter int source = -1; 6827e2ab150SChristoph Lameter int dest = 0; 6837e2ab150SChristoph Lameter 6847e2ab150SChristoph Lameter for_each_node_mask(s, tmp) { 6857e2ab150SChristoph Lameter d = node_remap(s, *from_nodes, *to_nodes); 6867e2ab150SChristoph Lameter if (s == d) 6877e2ab150SChristoph Lameter continue; 6887e2ab150SChristoph Lameter 6897e2ab150SChristoph Lameter source = s; /* Node moved. Memorize */ 6907e2ab150SChristoph Lameter dest = d; 6917e2ab150SChristoph Lameter 6927e2ab150SChristoph Lameter /* dest not in remaining from nodes? */ 6937e2ab150SChristoph Lameter if (!node_isset(dest, tmp)) 6947e2ab150SChristoph Lameter break; 6957e2ab150SChristoph Lameter } 6967e2ab150SChristoph Lameter if (source == -1) 6977e2ab150SChristoph Lameter break; 6987e2ab150SChristoph Lameter 6997e2ab150SChristoph Lameter node_clear(source, tmp); 7007e2ab150SChristoph Lameter err = migrate_to_node(mm, source, dest, flags); 7017e2ab150SChristoph Lameter if (err > 0) 7027e2ab150SChristoph Lameter busy += err; 7037e2ab150SChristoph Lameter if (err < 0) 7047e2ab150SChristoph Lameter break; 70539743889SChristoph Lameter } 706d4984711SChristoph Lameter 70739743889SChristoph Lameter up_read(&mm->mmap_sem); 7087e2ab150SChristoph Lameter if (err < 0) 7097e2ab150SChristoph Lameter return err; 7107e2ab150SChristoph Lameter return busy; 71139743889SChristoph Lameter } 71239743889SChristoph Lameter 7136ce3c4c0SChristoph Lameter long do_mbind(unsigned long start, unsigned long len, 7146ce3c4c0SChristoph Lameter unsigned long mode, nodemask_t *nmask, unsigned long flags) 7156ce3c4c0SChristoph Lameter { 7166ce3c4c0SChristoph Lameter struct vm_area_struct *vma; 7176ce3c4c0SChristoph Lameter struct mm_struct *mm = current->mm; 7186ce3c4c0SChristoph Lameter struct mempolicy *new; 7196ce3c4c0SChristoph Lameter unsigned long end; 7206ce3c4c0SChristoph Lameter int err; 7216ce3c4c0SChristoph Lameter LIST_HEAD(pagelist); 7226ce3c4c0SChristoph Lameter 7236ce3c4c0SChristoph Lameter if ((flags & ~(unsigned long)(MPOL_MF_STRICT | 7246ce3c4c0SChristoph Lameter MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) 7256ce3c4c0SChristoph Lameter || mode > MPOL_MAX) 7266ce3c4c0SChristoph Lameter return -EINVAL; 7276ce3c4c0SChristoph Lameter if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_RESOURCE)) 7286ce3c4c0SChristoph Lameter return -EPERM; 7296ce3c4c0SChristoph Lameter 7306ce3c4c0SChristoph Lameter if (start & ~PAGE_MASK) 7316ce3c4c0SChristoph Lameter return -EINVAL; 7326ce3c4c0SChristoph Lameter 7336ce3c4c0SChristoph Lameter if (mode == MPOL_DEFAULT) 7346ce3c4c0SChristoph Lameter flags &= ~MPOL_MF_STRICT; 7356ce3c4c0SChristoph Lameter 7366ce3c4c0SChristoph Lameter len = (len + PAGE_SIZE - 1) & PAGE_MASK; 7376ce3c4c0SChristoph Lameter end = start + len; 7386ce3c4c0SChristoph Lameter 7396ce3c4c0SChristoph Lameter if (end < start) 7406ce3c4c0SChristoph Lameter return -EINVAL; 7416ce3c4c0SChristoph Lameter if (end == start) 7426ce3c4c0SChristoph Lameter return 0; 7436ce3c4c0SChristoph Lameter 7446ce3c4c0SChristoph Lameter if (mpol_check_policy(mode, nmask)) 7456ce3c4c0SChristoph Lameter return -EINVAL; 7466ce3c4c0SChristoph Lameter 7476ce3c4c0SChristoph Lameter new = mpol_new(mode, nmask); 7486ce3c4c0SChristoph Lameter if (IS_ERR(new)) 7496ce3c4c0SChristoph Lameter return PTR_ERR(new); 7506ce3c4c0SChristoph Lameter 7516ce3c4c0SChristoph Lameter /* 7526ce3c4c0SChristoph Lameter * If we are using the default policy then operation 7536ce3c4c0SChristoph Lameter * on discontinuous address spaces is okay after all 7546ce3c4c0SChristoph Lameter */ 7556ce3c4c0SChristoph Lameter if (!new) 7566ce3c4c0SChristoph Lameter flags |= MPOL_MF_DISCONTIG_OK; 7576ce3c4c0SChristoph Lameter 7586ce3c4c0SChristoph Lameter PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len, 7596ce3c4c0SChristoph Lameter mode,nodes_addr(nodes)[0]); 7606ce3c4c0SChristoph Lameter 7616ce3c4c0SChristoph Lameter down_write(&mm->mmap_sem); 7626ce3c4c0SChristoph Lameter vma = check_range(mm, start, end, nmask, 7636ce3c4c0SChristoph Lameter flags | MPOL_MF_INVERT, &pagelist); 7646ce3c4c0SChristoph Lameter 7656ce3c4c0SChristoph Lameter err = PTR_ERR(vma); 7666ce3c4c0SChristoph Lameter if (!IS_ERR(vma)) { 7676ce3c4c0SChristoph Lameter int nr_failed = 0; 7686ce3c4c0SChristoph Lameter 7696ce3c4c0SChristoph Lameter err = mbind_range(vma, start, end, new); 7707e2ab150SChristoph Lameter 7716ce3c4c0SChristoph Lameter if (!list_empty(&pagelist)) 7727e2ab150SChristoph Lameter nr_failed = migrate_pages_to(&pagelist, vma, -1); 7736ce3c4c0SChristoph Lameter 7746ce3c4c0SChristoph Lameter if (!err && nr_failed && (flags & MPOL_MF_STRICT)) 7756ce3c4c0SChristoph Lameter err = -EIO; 7766ce3c4c0SChristoph Lameter } 7776ce3c4c0SChristoph Lameter if (!list_empty(&pagelist)) 7786ce3c4c0SChristoph Lameter putback_lru_pages(&pagelist); 7796ce3c4c0SChristoph Lameter 7806ce3c4c0SChristoph Lameter up_write(&mm->mmap_sem); 7816ce3c4c0SChristoph Lameter mpol_free(new); 7826ce3c4c0SChristoph Lameter return err; 7836ce3c4c0SChristoph Lameter } 7846ce3c4c0SChristoph Lameter 78539743889SChristoph Lameter /* 7868bccd85fSChristoph Lameter * User space interface with variable sized bitmaps for nodelists. 7878bccd85fSChristoph Lameter */ 7888bccd85fSChristoph Lameter 7898bccd85fSChristoph Lameter /* Copy a node mask from user space. */ 79039743889SChristoph Lameter static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask, 7918bccd85fSChristoph Lameter unsigned long maxnode) 7928bccd85fSChristoph Lameter { 7938bccd85fSChristoph Lameter unsigned long k; 7948bccd85fSChristoph Lameter unsigned long nlongs; 7958bccd85fSChristoph Lameter unsigned long endmask; 7968bccd85fSChristoph Lameter 7978bccd85fSChristoph Lameter --maxnode; 7988bccd85fSChristoph Lameter nodes_clear(*nodes); 7998bccd85fSChristoph Lameter if (maxnode == 0 || !nmask) 8008bccd85fSChristoph Lameter return 0; 8018bccd85fSChristoph Lameter 8028bccd85fSChristoph Lameter nlongs = BITS_TO_LONGS(maxnode); 8038bccd85fSChristoph Lameter if ((maxnode % BITS_PER_LONG) == 0) 8048bccd85fSChristoph Lameter endmask = ~0UL; 8058bccd85fSChristoph Lameter else 8068bccd85fSChristoph Lameter endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1; 8078bccd85fSChristoph Lameter 8088bccd85fSChristoph Lameter /* When the user specified more nodes than supported just check 8098bccd85fSChristoph Lameter if the non supported part is all zero. */ 8108bccd85fSChristoph Lameter if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) { 8118bccd85fSChristoph Lameter if (nlongs > PAGE_SIZE/sizeof(long)) 8128bccd85fSChristoph Lameter return -EINVAL; 8138bccd85fSChristoph Lameter for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) { 8148bccd85fSChristoph Lameter unsigned long t; 8158bccd85fSChristoph Lameter if (get_user(t, nmask + k)) 8168bccd85fSChristoph Lameter return -EFAULT; 8178bccd85fSChristoph Lameter if (k == nlongs - 1) { 8188bccd85fSChristoph Lameter if (t & endmask) 8198bccd85fSChristoph Lameter return -EINVAL; 8208bccd85fSChristoph Lameter } else if (t) 8218bccd85fSChristoph Lameter return -EINVAL; 8228bccd85fSChristoph Lameter } 8238bccd85fSChristoph Lameter nlongs = BITS_TO_LONGS(MAX_NUMNODES); 8248bccd85fSChristoph Lameter endmask = ~0UL; 8258bccd85fSChristoph Lameter } 8268bccd85fSChristoph Lameter 8278bccd85fSChristoph Lameter if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long))) 8288bccd85fSChristoph Lameter return -EFAULT; 8298bccd85fSChristoph Lameter nodes_addr(*nodes)[nlongs-1] &= endmask; 8308bccd85fSChristoph Lameter return 0; 8318bccd85fSChristoph Lameter } 8328bccd85fSChristoph Lameter 8338bccd85fSChristoph Lameter /* Copy a kernel node mask to user space */ 8348bccd85fSChristoph Lameter static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode, 8358bccd85fSChristoph Lameter nodemask_t *nodes) 8368bccd85fSChristoph Lameter { 8378bccd85fSChristoph Lameter unsigned long copy = ALIGN(maxnode-1, 64) / 8; 8388bccd85fSChristoph Lameter const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long); 8398bccd85fSChristoph Lameter 8408bccd85fSChristoph Lameter if (copy > nbytes) { 8418bccd85fSChristoph Lameter if (copy > PAGE_SIZE) 8428bccd85fSChristoph Lameter return -EINVAL; 8438bccd85fSChristoph Lameter if (clear_user((char __user *)mask + nbytes, copy - nbytes)) 8448bccd85fSChristoph Lameter return -EFAULT; 8458bccd85fSChristoph Lameter copy = nbytes; 8468bccd85fSChristoph Lameter } 8478bccd85fSChristoph Lameter return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0; 8488bccd85fSChristoph Lameter } 8498bccd85fSChristoph Lameter 8508bccd85fSChristoph Lameter asmlinkage long sys_mbind(unsigned long start, unsigned long len, 8518bccd85fSChristoph Lameter unsigned long mode, 8528bccd85fSChristoph Lameter unsigned long __user *nmask, unsigned long maxnode, 8538bccd85fSChristoph Lameter unsigned flags) 8548bccd85fSChristoph Lameter { 8558bccd85fSChristoph Lameter nodemask_t nodes; 8568bccd85fSChristoph Lameter int err; 8578bccd85fSChristoph Lameter 8588bccd85fSChristoph Lameter err = get_nodes(&nodes, nmask, maxnode); 8598bccd85fSChristoph Lameter if (err) 8608bccd85fSChristoph Lameter return err; 8618bccd85fSChristoph Lameter return do_mbind(start, len, mode, &nodes, flags); 8628bccd85fSChristoph Lameter } 8638bccd85fSChristoph Lameter 8648bccd85fSChristoph Lameter /* Set the process memory policy */ 8658bccd85fSChristoph Lameter asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask, 8668bccd85fSChristoph Lameter unsigned long maxnode) 8678bccd85fSChristoph Lameter { 8688bccd85fSChristoph Lameter int err; 8698bccd85fSChristoph Lameter nodemask_t nodes; 8708bccd85fSChristoph Lameter 8718bccd85fSChristoph Lameter if (mode < 0 || mode > MPOL_MAX) 8728bccd85fSChristoph Lameter return -EINVAL; 8738bccd85fSChristoph Lameter err = get_nodes(&nodes, nmask, maxnode); 8748bccd85fSChristoph Lameter if (err) 8758bccd85fSChristoph Lameter return err; 8768bccd85fSChristoph Lameter return do_set_mempolicy(mode, &nodes); 8778bccd85fSChristoph Lameter } 8788bccd85fSChristoph Lameter 87939743889SChristoph Lameter asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode, 88039743889SChristoph Lameter const unsigned long __user *old_nodes, 88139743889SChristoph Lameter const unsigned long __user *new_nodes) 88239743889SChristoph Lameter { 88339743889SChristoph Lameter struct mm_struct *mm; 88439743889SChristoph Lameter struct task_struct *task; 88539743889SChristoph Lameter nodemask_t old; 88639743889SChristoph Lameter nodemask_t new; 88739743889SChristoph Lameter nodemask_t task_nodes; 88839743889SChristoph Lameter int err; 88939743889SChristoph Lameter 89039743889SChristoph Lameter err = get_nodes(&old, old_nodes, maxnode); 89139743889SChristoph Lameter if (err) 89239743889SChristoph Lameter return err; 89339743889SChristoph Lameter 89439743889SChristoph Lameter err = get_nodes(&new, new_nodes, maxnode); 89539743889SChristoph Lameter if (err) 89639743889SChristoph Lameter return err; 89739743889SChristoph Lameter 89839743889SChristoph Lameter /* Find the mm_struct */ 89939743889SChristoph Lameter read_lock(&tasklist_lock); 90039743889SChristoph Lameter task = pid ? find_task_by_pid(pid) : current; 90139743889SChristoph Lameter if (!task) { 90239743889SChristoph Lameter read_unlock(&tasklist_lock); 90339743889SChristoph Lameter return -ESRCH; 90439743889SChristoph Lameter } 90539743889SChristoph Lameter mm = get_task_mm(task); 90639743889SChristoph Lameter read_unlock(&tasklist_lock); 90739743889SChristoph Lameter 90839743889SChristoph Lameter if (!mm) 90939743889SChristoph Lameter return -EINVAL; 91039743889SChristoph Lameter 91139743889SChristoph Lameter /* 91239743889SChristoph Lameter * Check if this process has the right to modify the specified 91339743889SChristoph Lameter * process. The right exists if the process has administrative 91439743889SChristoph Lameter * capabilities, superuser priviledges or the same 91539743889SChristoph Lameter * userid as the target process. 91639743889SChristoph Lameter */ 91739743889SChristoph Lameter if ((current->euid != task->suid) && (current->euid != task->uid) && 91839743889SChristoph Lameter (current->uid != task->suid) && (current->uid != task->uid) && 91939743889SChristoph Lameter !capable(CAP_SYS_ADMIN)) { 92039743889SChristoph Lameter err = -EPERM; 92139743889SChristoph Lameter goto out; 92239743889SChristoph Lameter } 92339743889SChristoph Lameter 92439743889SChristoph Lameter task_nodes = cpuset_mems_allowed(task); 92539743889SChristoph Lameter /* Is the user allowed to access the target nodes? */ 92639743889SChristoph Lameter if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_ADMIN)) { 92739743889SChristoph Lameter err = -EPERM; 92839743889SChristoph Lameter goto out; 92939743889SChristoph Lameter } 93039743889SChristoph Lameter 93139743889SChristoph Lameter err = do_migrate_pages(mm, &old, &new, MPOL_MF_MOVE); 93239743889SChristoph Lameter out: 93339743889SChristoph Lameter mmput(mm); 93439743889SChristoph Lameter return err; 93539743889SChristoph Lameter } 93639743889SChristoph Lameter 93739743889SChristoph Lameter 9388bccd85fSChristoph Lameter /* Retrieve NUMA policy */ 9398bccd85fSChristoph Lameter asmlinkage long sys_get_mempolicy(int __user *policy, 9408bccd85fSChristoph Lameter unsigned long __user *nmask, 9418bccd85fSChristoph Lameter unsigned long maxnode, 9428bccd85fSChristoph Lameter unsigned long addr, unsigned long flags) 9438bccd85fSChristoph Lameter { 9448bccd85fSChristoph Lameter int err, pval; 9458bccd85fSChristoph Lameter nodemask_t nodes; 9468bccd85fSChristoph Lameter 9478bccd85fSChristoph Lameter if (nmask != NULL && maxnode < MAX_NUMNODES) 9488bccd85fSChristoph Lameter return -EINVAL; 9498bccd85fSChristoph Lameter 9508bccd85fSChristoph Lameter err = do_get_mempolicy(&pval, &nodes, addr, flags); 9518bccd85fSChristoph Lameter 9528bccd85fSChristoph Lameter if (err) 9538bccd85fSChristoph Lameter return err; 9548bccd85fSChristoph Lameter 9558bccd85fSChristoph Lameter if (policy && put_user(pval, policy)) 9568bccd85fSChristoph Lameter return -EFAULT; 9578bccd85fSChristoph Lameter 9588bccd85fSChristoph Lameter if (nmask) 9598bccd85fSChristoph Lameter err = copy_nodes_to_user(nmask, maxnode, &nodes); 9608bccd85fSChristoph Lameter 9618bccd85fSChristoph Lameter return err; 9628bccd85fSChristoph Lameter } 9638bccd85fSChristoph Lameter 9641da177e4SLinus Torvalds #ifdef CONFIG_COMPAT 9651da177e4SLinus Torvalds 9661da177e4SLinus Torvalds asmlinkage long compat_sys_get_mempolicy(int __user *policy, 9671da177e4SLinus Torvalds compat_ulong_t __user *nmask, 9681da177e4SLinus Torvalds compat_ulong_t maxnode, 9691da177e4SLinus Torvalds compat_ulong_t addr, compat_ulong_t flags) 9701da177e4SLinus Torvalds { 9711da177e4SLinus Torvalds long err; 9721da177e4SLinus Torvalds unsigned long __user *nm = NULL; 9731da177e4SLinus Torvalds unsigned long nr_bits, alloc_size; 9741da177e4SLinus Torvalds DECLARE_BITMAP(bm, MAX_NUMNODES); 9751da177e4SLinus Torvalds 9761da177e4SLinus Torvalds nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); 9771da177e4SLinus Torvalds alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; 9781da177e4SLinus Torvalds 9791da177e4SLinus Torvalds if (nmask) 9801da177e4SLinus Torvalds nm = compat_alloc_user_space(alloc_size); 9811da177e4SLinus Torvalds 9821da177e4SLinus Torvalds err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags); 9831da177e4SLinus Torvalds 9841da177e4SLinus Torvalds if (!err && nmask) { 9851da177e4SLinus Torvalds err = copy_from_user(bm, nm, alloc_size); 9861da177e4SLinus Torvalds /* ensure entire bitmap is zeroed */ 9871da177e4SLinus Torvalds err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8); 9881da177e4SLinus Torvalds err |= compat_put_bitmap(nmask, bm, nr_bits); 9891da177e4SLinus Torvalds } 9901da177e4SLinus Torvalds 9911da177e4SLinus Torvalds return err; 9921da177e4SLinus Torvalds } 9931da177e4SLinus Torvalds 9941da177e4SLinus Torvalds asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask, 9951da177e4SLinus Torvalds compat_ulong_t maxnode) 9961da177e4SLinus Torvalds { 9971da177e4SLinus Torvalds long err = 0; 9981da177e4SLinus Torvalds unsigned long __user *nm = NULL; 9991da177e4SLinus Torvalds unsigned long nr_bits, alloc_size; 10001da177e4SLinus Torvalds DECLARE_BITMAP(bm, MAX_NUMNODES); 10011da177e4SLinus Torvalds 10021da177e4SLinus Torvalds nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); 10031da177e4SLinus Torvalds alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; 10041da177e4SLinus Torvalds 10051da177e4SLinus Torvalds if (nmask) { 10061da177e4SLinus Torvalds err = compat_get_bitmap(bm, nmask, nr_bits); 10071da177e4SLinus Torvalds nm = compat_alloc_user_space(alloc_size); 10081da177e4SLinus Torvalds err |= copy_to_user(nm, bm, alloc_size); 10091da177e4SLinus Torvalds } 10101da177e4SLinus Torvalds 10111da177e4SLinus Torvalds if (err) 10121da177e4SLinus Torvalds return -EFAULT; 10131da177e4SLinus Torvalds 10141da177e4SLinus Torvalds return sys_set_mempolicy(mode, nm, nr_bits+1); 10151da177e4SLinus Torvalds } 10161da177e4SLinus Torvalds 10171da177e4SLinus Torvalds asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len, 10181da177e4SLinus Torvalds compat_ulong_t mode, compat_ulong_t __user *nmask, 10191da177e4SLinus Torvalds compat_ulong_t maxnode, compat_ulong_t flags) 10201da177e4SLinus Torvalds { 10211da177e4SLinus Torvalds long err = 0; 10221da177e4SLinus Torvalds unsigned long __user *nm = NULL; 10231da177e4SLinus Torvalds unsigned long nr_bits, alloc_size; 1024dfcd3c0dSAndi Kleen nodemask_t bm; 10251da177e4SLinus Torvalds 10261da177e4SLinus Torvalds nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); 10271da177e4SLinus Torvalds alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; 10281da177e4SLinus Torvalds 10291da177e4SLinus Torvalds if (nmask) { 1030dfcd3c0dSAndi Kleen err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits); 10311da177e4SLinus Torvalds nm = compat_alloc_user_space(alloc_size); 1032dfcd3c0dSAndi Kleen err |= copy_to_user(nm, nodes_addr(bm), alloc_size); 10331da177e4SLinus Torvalds } 10341da177e4SLinus Torvalds 10351da177e4SLinus Torvalds if (err) 10361da177e4SLinus Torvalds return -EFAULT; 10371da177e4SLinus Torvalds 10381da177e4SLinus Torvalds return sys_mbind(start, len, mode, nm, nr_bits+1, flags); 10391da177e4SLinus Torvalds } 10401da177e4SLinus Torvalds 10411da177e4SLinus Torvalds #endif 10421da177e4SLinus Torvalds 10431da177e4SLinus Torvalds /* Return effective policy for a VMA */ 104448fce342SChristoph Lameter static struct mempolicy * get_vma_policy(struct task_struct *task, 104548fce342SChristoph Lameter struct vm_area_struct *vma, unsigned long addr) 10461da177e4SLinus Torvalds { 10476e21c8f1SChristoph Lameter struct mempolicy *pol = task->mempolicy; 10481da177e4SLinus Torvalds 10491da177e4SLinus Torvalds if (vma) { 10501da177e4SLinus Torvalds if (vma->vm_ops && vma->vm_ops->get_policy) 10511da177e4SLinus Torvalds pol = vma->vm_ops->get_policy(vma, addr); 10521da177e4SLinus Torvalds else if (vma->vm_policy && 10531da177e4SLinus Torvalds vma->vm_policy->policy != MPOL_DEFAULT) 10541da177e4SLinus Torvalds pol = vma->vm_policy; 10551da177e4SLinus Torvalds } 10561da177e4SLinus Torvalds if (!pol) 10571da177e4SLinus Torvalds pol = &default_policy; 10581da177e4SLinus Torvalds return pol; 10591da177e4SLinus Torvalds } 10601da177e4SLinus Torvalds 10611da177e4SLinus Torvalds /* Return a zonelist representing a mempolicy */ 1062dd0fc66fSAl Viro static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy) 10631da177e4SLinus Torvalds { 10641da177e4SLinus Torvalds int nd; 10651da177e4SLinus Torvalds 10661da177e4SLinus Torvalds switch (policy->policy) { 10671da177e4SLinus Torvalds case MPOL_PREFERRED: 10681da177e4SLinus Torvalds nd = policy->v.preferred_node; 10691da177e4SLinus Torvalds if (nd < 0) 10701da177e4SLinus Torvalds nd = numa_node_id(); 10711da177e4SLinus Torvalds break; 10721da177e4SLinus Torvalds case MPOL_BIND: 10731da177e4SLinus Torvalds /* Lower zones don't get a policy applied */ 10741da177e4SLinus Torvalds /* Careful: current->mems_allowed might have moved */ 1075af4ca457SAl Viro if (gfp_zone(gfp) >= policy_zone) 10761da177e4SLinus Torvalds if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist)) 10771da177e4SLinus Torvalds return policy->v.zonelist; 10781da177e4SLinus Torvalds /*FALL THROUGH*/ 10791da177e4SLinus Torvalds case MPOL_INTERLEAVE: /* should not happen */ 10801da177e4SLinus Torvalds case MPOL_DEFAULT: 10811da177e4SLinus Torvalds nd = numa_node_id(); 10821da177e4SLinus Torvalds break; 10831da177e4SLinus Torvalds default: 10841da177e4SLinus Torvalds nd = 0; 10851da177e4SLinus Torvalds BUG(); 10861da177e4SLinus Torvalds } 1087af4ca457SAl Viro return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp); 10881da177e4SLinus Torvalds } 10891da177e4SLinus Torvalds 10901da177e4SLinus Torvalds /* Do dynamic interleaving for a process */ 10911da177e4SLinus Torvalds static unsigned interleave_nodes(struct mempolicy *policy) 10921da177e4SLinus Torvalds { 10931da177e4SLinus Torvalds unsigned nid, next; 10941da177e4SLinus Torvalds struct task_struct *me = current; 10951da177e4SLinus Torvalds 10961da177e4SLinus Torvalds nid = me->il_next; 1097dfcd3c0dSAndi Kleen next = next_node(nid, policy->v.nodes); 10981da177e4SLinus Torvalds if (next >= MAX_NUMNODES) 1099dfcd3c0dSAndi Kleen next = first_node(policy->v.nodes); 11001da177e4SLinus Torvalds me->il_next = next; 11011da177e4SLinus Torvalds return nid; 11021da177e4SLinus Torvalds } 11031da177e4SLinus Torvalds 1104dc85da15SChristoph Lameter /* 1105dc85da15SChristoph Lameter * Depending on the memory policy provide a node from which to allocate the 1106dc85da15SChristoph Lameter * next slab entry. 1107dc85da15SChristoph Lameter */ 1108dc85da15SChristoph Lameter unsigned slab_node(struct mempolicy *policy) 1109dc85da15SChristoph Lameter { 1110dc85da15SChristoph Lameter switch (policy->policy) { 1111dc85da15SChristoph Lameter case MPOL_INTERLEAVE: 1112dc85da15SChristoph Lameter return interleave_nodes(policy); 1113dc85da15SChristoph Lameter 1114dc85da15SChristoph Lameter case MPOL_BIND: 1115dc85da15SChristoph Lameter /* 1116dc85da15SChristoph Lameter * Follow bind policy behavior and start allocation at the 1117dc85da15SChristoph Lameter * first node. 1118dc85da15SChristoph Lameter */ 1119dc85da15SChristoph Lameter return policy->v.zonelist->zones[0]->zone_pgdat->node_id; 1120dc85da15SChristoph Lameter 1121dc85da15SChristoph Lameter case MPOL_PREFERRED: 1122dc85da15SChristoph Lameter if (policy->v.preferred_node >= 0) 1123dc85da15SChristoph Lameter return policy->v.preferred_node; 1124dc85da15SChristoph Lameter /* Fall through */ 1125dc85da15SChristoph Lameter 1126dc85da15SChristoph Lameter default: 1127dc85da15SChristoph Lameter return numa_node_id(); 1128dc85da15SChristoph Lameter } 1129dc85da15SChristoph Lameter } 1130dc85da15SChristoph Lameter 11311da177e4SLinus Torvalds /* Do static interleaving for a VMA with known offset. */ 11321da177e4SLinus Torvalds static unsigned offset_il_node(struct mempolicy *pol, 11331da177e4SLinus Torvalds struct vm_area_struct *vma, unsigned long off) 11341da177e4SLinus Torvalds { 1135dfcd3c0dSAndi Kleen unsigned nnodes = nodes_weight(pol->v.nodes); 11361da177e4SLinus Torvalds unsigned target = (unsigned)off % nnodes; 11371da177e4SLinus Torvalds int c; 11381da177e4SLinus Torvalds int nid = -1; 11391da177e4SLinus Torvalds 11401da177e4SLinus Torvalds c = 0; 11411da177e4SLinus Torvalds do { 1142dfcd3c0dSAndi Kleen nid = next_node(nid, pol->v.nodes); 11431da177e4SLinus Torvalds c++; 11441da177e4SLinus Torvalds } while (c <= target); 11451da177e4SLinus Torvalds return nid; 11461da177e4SLinus Torvalds } 11471da177e4SLinus Torvalds 11485da7ca86SChristoph Lameter /* Determine a node number for interleave */ 11495da7ca86SChristoph Lameter static inline unsigned interleave_nid(struct mempolicy *pol, 11505da7ca86SChristoph Lameter struct vm_area_struct *vma, unsigned long addr, int shift) 11515da7ca86SChristoph Lameter { 11525da7ca86SChristoph Lameter if (vma) { 11535da7ca86SChristoph Lameter unsigned long off; 11545da7ca86SChristoph Lameter 11555da7ca86SChristoph Lameter off = vma->vm_pgoff; 11565da7ca86SChristoph Lameter off += (addr - vma->vm_start) >> shift; 11575da7ca86SChristoph Lameter return offset_il_node(pol, vma, off); 11585da7ca86SChristoph Lameter } else 11595da7ca86SChristoph Lameter return interleave_nodes(pol); 11605da7ca86SChristoph Lameter } 11615da7ca86SChristoph Lameter 1162*00ac59adSChen, Kenneth W #ifdef CONFIG_HUGETLBFS 11635da7ca86SChristoph Lameter /* Return a zonelist suitable for a huge page allocation. */ 11645da7ca86SChristoph Lameter struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr) 11655da7ca86SChristoph Lameter { 11665da7ca86SChristoph Lameter struct mempolicy *pol = get_vma_policy(current, vma, addr); 11675da7ca86SChristoph Lameter 11685da7ca86SChristoph Lameter if (pol->policy == MPOL_INTERLEAVE) { 11695da7ca86SChristoph Lameter unsigned nid; 11705da7ca86SChristoph Lameter 11715da7ca86SChristoph Lameter nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT); 11725da7ca86SChristoph Lameter return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER); 11735da7ca86SChristoph Lameter } 11745da7ca86SChristoph Lameter return zonelist_policy(GFP_HIGHUSER, pol); 11755da7ca86SChristoph Lameter } 1176*00ac59adSChen, Kenneth W #endif 11775da7ca86SChristoph Lameter 11781da177e4SLinus Torvalds /* Allocate a page in interleaved policy. 11791da177e4SLinus Torvalds Own path because it needs to do special accounting. */ 1180662f3a0bSAndi Kleen static struct page *alloc_page_interleave(gfp_t gfp, unsigned order, 1181662f3a0bSAndi Kleen unsigned nid) 11821da177e4SLinus Torvalds { 11831da177e4SLinus Torvalds struct zonelist *zl; 11841da177e4SLinus Torvalds struct page *page; 11851da177e4SLinus Torvalds 1186af4ca457SAl Viro zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp); 11871da177e4SLinus Torvalds page = __alloc_pages(gfp, order, zl); 11881da177e4SLinus Torvalds if (page && page_zone(page) == zl->zones[0]) { 1189e7c8d5c9SChristoph Lameter zone_pcp(zl->zones[0],get_cpu())->interleave_hit++; 11901da177e4SLinus Torvalds put_cpu(); 11911da177e4SLinus Torvalds } 11921da177e4SLinus Torvalds return page; 11931da177e4SLinus Torvalds } 11941da177e4SLinus Torvalds 11951da177e4SLinus Torvalds /** 11961da177e4SLinus Torvalds * alloc_page_vma - Allocate a page for a VMA. 11971da177e4SLinus Torvalds * 11981da177e4SLinus Torvalds * @gfp: 11991da177e4SLinus Torvalds * %GFP_USER user allocation. 12001da177e4SLinus Torvalds * %GFP_KERNEL kernel allocations, 12011da177e4SLinus Torvalds * %GFP_HIGHMEM highmem/user allocations, 12021da177e4SLinus Torvalds * %GFP_FS allocation should not call back into a file system. 12031da177e4SLinus Torvalds * %GFP_ATOMIC don't sleep. 12041da177e4SLinus Torvalds * 12051da177e4SLinus Torvalds * @vma: Pointer to VMA or NULL if not available. 12061da177e4SLinus Torvalds * @addr: Virtual Address of the allocation. Must be inside the VMA. 12071da177e4SLinus Torvalds * 12081da177e4SLinus Torvalds * This function allocates a page from the kernel page pool and applies 12091da177e4SLinus Torvalds * a NUMA policy associated with the VMA or the current process. 12101da177e4SLinus Torvalds * When VMA is not NULL caller must hold down_read on the mmap_sem of the 12111da177e4SLinus Torvalds * mm_struct of the VMA to prevent it from going away. Should be used for 12121da177e4SLinus Torvalds * all allocations for pages that will be mapped into 12131da177e4SLinus Torvalds * user space. Returns NULL when no page can be allocated. 12141da177e4SLinus Torvalds * 12151da177e4SLinus Torvalds * Should be called with the mm_sem of the vma hold. 12161da177e4SLinus Torvalds */ 12171da177e4SLinus Torvalds struct page * 1218dd0fc66fSAl Viro alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) 12191da177e4SLinus Torvalds { 12206e21c8f1SChristoph Lameter struct mempolicy *pol = get_vma_policy(current, vma, addr); 12211da177e4SLinus Torvalds 1222cf2a473cSPaul Jackson cpuset_update_task_memory_state(); 12231da177e4SLinus Torvalds 12241da177e4SLinus Torvalds if (unlikely(pol->policy == MPOL_INTERLEAVE)) { 12251da177e4SLinus Torvalds unsigned nid; 12265da7ca86SChristoph Lameter 12275da7ca86SChristoph Lameter nid = interleave_nid(pol, vma, addr, PAGE_SHIFT); 12281da177e4SLinus Torvalds return alloc_page_interleave(gfp, 0, nid); 12291da177e4SLinus Torvalds } 12301da177e4SLinus Torvalds return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol)); 12311da177e4SLinus Torvalds } 12321da177e4SLinus Torvalds 12331da177e4SLinus Torvalds /** 12341da177e4SLinus Torvalds * alloc_pages_current - Allocate pages. 12351da177e4SLinus Torvalds * 12361da177e4SLinus Torvalds * @gfp: 12371da177e4SLinus Torvalds * %GFP_USER user allocation, 12381da177e4SLinus Torvalds * %GFP_KERNEL kernel allocation, 12391da177e4SLinus Torvalds * %GFP_HIGHMEM highmem allocation, 12401da177e4SLinus Torvalds * %GFP_FS don't call back into a file system. 12411da177e4SLinus Torvalds * %GFP_ATOMIC don't sleep. 12421da177e4SLinus Torvalds * @order: Power of two of allocation size in pages. 0 is a single page. 12431da177e4SLinus Torvalds * 12441da177e4SLinus Torvalds * Allocate a page from the kernel page pool. When not in 12451da177e4SLinus Torvalds * interrupt context and apply the current process NUMA policy. 12461da177e4SLinus Torvalds * Returns NULL when no page can be allocated. 12471da177e4SLinus Torvalds * 1248cf2a473cSPaul Jackson * Don't call cpuset_update_task_memory_state() unless 12491da177e4SLinus Torvalds * 1) it's ok to take cpuset_sem (can WAIT), and 12501da177e4SLinus Torvalds * 2) allocating for current task (not interrupt). 12511da177e4SLinus Torvalds */ 1252dd0fc66fSAl Viro struct page *alloc_pages_current(gfp_t gfp, unsigned order) 12531da177e4SLinus Torvalds { 12541da177e4SLinus Torvalds struct mempolicy *pol = current->mempolicy; 12551da177e4SLinus Torvalds 12561da177e4SLinus Torvalds if ((gfp & __GFP_WAIT) && !in_interrupt()) 1257cf2a473cSPaul Jackson cpuset_update_task_memory_state(); 12581da177e4SLinus Torvalds if (!pol || in_interrupt()) 12591da177e4SLinus Torvalds pol = &default_policy; 12601da177e4SLinus Torvalds if (pol->policy == MPOL_INTERLEAVE) 12611da177e4SLinus Torvalds return alloc_page_interleave(gfp, order, interleave_nodes(pol)); 12621da177e4SLinus Torvalds return __alloc_pages(gfp, order, zonelist_policy(gfp, pol)); 12631da177e4SLinus Torvalds } 12641da177e4SLinus Torvalds EXPORT_SYMBOL(alloc_pages_current); 12651da177e4SLinus Torvalds 12664225399aSPaul Jackson /* 12674225399aSPaul Jackson * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it 12684225399aSPaul Jackson * rebinds the mempolicy its copying by calling mpol_rebind_policy() 12694225399aSPaul Jackson * with the mems_allowed returned by cpuset_mems_allowed(). This 12704225399aSPaul Jackson * keeps mempolicies cpuset relative after its cpuset moves. See 12714225399aSPaul Jackson * further kernel/cpuset.c update_nodemask(). 12724225399aSPaul Jackson */ 12734225399aSPaul Jackson void *cpuset_being_rebound; 12744225399aSPaul Jackson 12751da177e4SLinus Torvalds /* Slow path of a mempolicy copy */ 12761da177e4SLinus Torvalds struct mempolicy *__mpol_copy(struct mempolicy *old) 12771da177e4SLinus Torvalds { 12781da177e4SLinus Torvalds struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL); 12791da177e4SLinus Torvalds 12801da177e4SLinus Torvalds if (!new) 12811da177e4SLinus Torvalds return ERR_PTR(-ENOMEM); 12824225399aSPaul Jackson if (current_cpuset_is_being_rebound()) { 12834225399aSPaul Jackson nodemask_t mems = cpuset_mems_allowed(current); 12844225399aSPaul Jackson mpol_rebind_policy(old, &mems); 12854225399aSPaul Jackson } 12861da177e4SLinus Torvalds *new = *old; 12871da177e4SLinus Torvalds atomic_set(&new->refcnt, 1); 12881da177e4SLinus Torvalds if (new->policy == MPOL_BIND) { 12891da177e4SLinus Torvalds int sz = ksize(old->v.zonelist); 12901da177e4SLinus Torvalds new->v.zonelist = kmalloc(sz, SLAB_KERNEL); 12911da177e4SLinus Torvalds if (!new->v.zonelist) { 12921da177e4SLinus Torvalds kmem_cache_free(policy_cache, new); 12931da177e4SLinus Torvalds return ERR_PTR(-ENOMEM); 12941da177e4SLinus Torvalds } 12951da177e4SLinus Torvalds memcpy(new->v.zonelist, old->v.zonelist, sz); 12961da177e4SLinus Torvalds } 12971da177e4SLinus Torvalds return new; 12981da177e4SLinus Torvalds } 12991da177e4SLinus Torvalds 13001da177e4SLinus Torvalds /* Slow path of a mempolicy comparison */ 13011da177e4SLinus Torvalds int __mpol_equal(struct mempolicy *a, struct mempolicy *b) 13021da177e4SLinus Torvalds { 13031da177e4SLinus Torvalds if (!a || !b) 13041da177e4SLinus Torvalds return 0; 13051da177e4SLinus Torvalds if (a->policy != b->policy) 13061da177e4SLinus Torvalds return 0; 13071da177e4SLinus Torvalds switch (a->policy) { 13081da177e4SLinus Torvalds case MPOL_DEFAULT: 13091da177e4SLinus Torvalds return 1; 13101da177e4SLinus Torvalds case MPOL_INTERLEAVE: 1311dfcd3c0dSAndi Kleen return nodes_equal(a->v.nodes, b->v.nodes); 13121da177e4SLinus Torvalds case MPOL_PREFERRED: 13131da177e4SLinus Torvalds return a->v.preferred_node == b->v.preferred_node; 13141da177e4SLinus Torvalds case MPOL_BIND: { 13151da177e4SLinus Torvalds int i; 13161da177e4SLinus Torvalds for (i = 0; a->v.zonelist->zones[i]; i++) 13171da177e4SLinus Torvalds if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i]) 13181da177e4SLinus Torvalds return 0; 13191da177e4SLinus Torvalds return b->v.zonelist->zones[i] == NULL; 13201da177e4SLinus Torvalds } 13211da177e4SLinus Torvalds default: 13221da177e4SLinus Torvalds BUG(); 13231da177e4SLinus Torvalds return 0; 13241da177e4SLinus Torvalds } 13251da177e4SLinus Torvalds } 13261da177e4SLinus Torvalds 13271da177e4SLinus Torvalds /* Slow path of a mpol destructor. */ 13281da177e4SLinus Torvalds void __mpol_free(struct mempolicy *p) 13291da177e4SLinus Torvalds { 13301da177e4SLinus Torvalds if (!atomic_dec_and_test(&p->refcnt)) 13311da177e4SLinus Torvalds return; 13321da177e4SLinus Torvalds if (p->policy == MPOL_BIND) 13331da177e4SLinus Torvalds kfree(p->v.zonelist); 13341da177e4SLinus Torvalds p->policy = MPOL_DEFAULT; 13351da177e4SLinus Torvalds kmem_cache_free(policy_cache, p); 13361da177e4SLinus Torvalds } 13371da177e4SLinus Torvalds 13381da177e4SLinus Torvalds /* 13391da177e4SLinus Torvalds * Shared memory backing store policy support. 13401da177e4SLinus Torvalds * 13411da177e4SLinus Torvalds * Remember policies even when nobody has shared memory mapped. 13421da177e4SLinus Torvalds * The policies are kept in Red-Black tree linked from the inode. 13431da177e4SLinus Torvalds * They are protected by the sp->lock spinlock, which should be held 13441da177e4SLinus Torvalds * for any accesses to the tree. 13451da177e4SLinus Torvalds */ 13461da177e4SLinus Torvalds 13471da177e4SLinus Torvalds /* lookup first element intersecting start-end */ 13481da177e4SLinus Torvalds /* Caller holds sp->lock */ 13491da177e4SLinus Torvalds static struct sp_node * 13501da177e4SLinus Torvalds sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end) 13511da177e4SLinus Torvalds { 13521da177e4SLinus Torvalds struct rb_node *n = sp->root.rb_node; 13531da177e4SLinus Torvalds 13541da177e4SLinus Torvalds while (n) { 13551da177e4SLinus Torvalds struct sp_node *p = rb_entry(n, struct sp_node, nd); 13561da177e4SLinus Torvalds 13571da177e4SLinus Torvalds if (start >= p->end) 13581da177e4SLinus Torvalds n = n->rb_right; 13591da177e4SLinus Torvalds else if (end <= p->start) 13601da177e4SLinus Torvalds n = n->rb_left; 13611da177e4SLinus Torvalds else 13621da177e4SLinus Torvalds break; 13631da177e4SLinus Torvalds } 13641da177e4SLinus Torvalds if (!n) 13651da177e4SLinus Torvalds return NULL; 13661da177e4SLinus Torvalds for (;;) { 13671da177e4SLinus Torvalds struct sp_node *w = NULL; 13681da177e4SLinus Torvalds struct rb_node *prev = rb_prev(n); 13691da177e4SLinus Torvalds if (!prev) 13701da177e4SLinus Torvalds break; 13711da177e4SLinus Torvalds w = rb_entry(prev, struct sp_node, nd); 13721da177e4SLinus Torvalds if (w->end <= start) 13731da177e4SLinus Torvalds break; 13741da177e4SLinus Torvalds n = prev; 13751da177e4SLinus Torvalds } 13761da177e4SLinus Torvalds return rb_entry(n, struct sp_node, nd); 13771da177e4SLinus Torvalds } 13781da177e4SLinus Torvalds 13791da177e4SLinus Torvalds /* Insert a new shared policy into the list. */ 13801da177e4SLinus Torvalds /* Caller holds sp->lock */ 13811da177e4SLinus Torvalds static void sp_insert(struct shared_policy *sp, struct sp_node *new) 13821da177e4SLinus Torvalds { 13831da177e4SLinus Torvalds struct rb_node **p = &sp->root.rb_node; 13841da177e4SLinus Torvalds struct rb_node *parent = NULL; 13851da177e4SLinus Torvalds struct sp_node *nd; 13861da177e4SLinus Torvalds 13871da177e4SLinus Torvalds while (*p) { 13881da177e4SLinus Torvalds parent = *p; 13891da177e4SLinus Torvalds nd = rb_entry(parent, struct sp_node, nd); 13901da177e4SLinus Torvalds if (new->start < nd->start) 13911da177e4SLinus Torvalds p = &(*p)->rb_left; 13921da177e4SLinus Torvalds else if (new->end > nd->end) 13931da177e4SLinus Torvalds p = &(*p)->rb_right; 13941da177e4SLinus Torvalds else 13951da177e4SLinus Torvalds BUG(); 13961da177e4SLinus Torvalds } 13971da177e4SLinus Torvalds rb_link_node(&new->nd, parent, p); 13981da177e4SLinus Torvalds rb_insert_color(&new->nd, &sp->root); 13991da177e4SLinus Torvalds PDprintk("inserting %lx-%lx: %d\n", new->start, new->end, 14001da177e4SLinus Torvalds new->policy ? new->policy->policy : 0); 14011da177e4SLinus Torvalds } 14021da177e4SLinus Torvalds 14031da177e4SLinus Torvalds /* Find shared policy intersecting idx */ 14041da177e4SLinus Torvalds struct mempolicy * 14051da177e4SLinus Torvalds mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx) 14061da177e4SLinus Torvalds { 14071da177e4SLinus Torvalds struct mempolicy *pol = NULL; 14081da177e4SLinus Torvalds struct sp_node *sn; 14091da177e4SLinus Torvalds 14101da177e4SLinus Torvalds if (!sp->root.rb_node) 14111da177e4SLinus Torvalds return NULL; 14121da177e4SLinus Torvalds spin_lock(&sp->lock); 14131da177e4SLinus Torvalds sn = sp_lookup(sp, idx, idx+1); 14141da177e4SLinus Torvalds if (sn) { 14151da177e4SLinus Torvalds mpol_get(sn->policy); 14161da177e4SLinus Torvalds pol = sn->policy; 14171da177e4SLinus Torvalds } 14181da177e4SLinus Torvalds spin_unlock(&sp->lock); 14191da177e4SLinus Torvalds return pol; 14201da177e4SLinus Torvalds } 14211da177e4SLinus Torvalds 14221da177e4SLinus Torvalds static void sp_delete(struct shared_policy *sp, struct sp_node *n) 14231da177e4SLinus Torvalds { 14241da177e4SLinus Torvalds PDprintk("deleting %lx-l%x\n", n->start, n->end); 14251da177e4SLinus Torvalds rb_erase(&n->nd, &sp->root); 14261da177e4SLinus Torvalds mpol_free(n->policy); 14271da177e4SLinus Torvalds kmem_cache_free(sn_cache, n); 14281da177e4SLinus Torvalds } 14291da177e4SLinus Torvalds 14301da177e4SLinus Torvalds struct sp_node * 14311da177e4SLinus Torvalds sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol) 14321da177e4SLinus Torvalds { 14331da177e4SLinus Torvalds struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL); 14341da177e4SLinus Torvalds 14351da177e4SLinus Torvalds if (!n) 14361da177e4SLinus Torvalds return NULL; 14371da177e4SLinus Torvalds n->start = start; 14381da177e4SLinus Torvalds n->end = end; 14391da177e4SLinus Torvalds mpol_get(pol); 14401da177e4SLinus Torvalds n->policy = pol; 14411da177e4SLinus Torvalds return n; 14421da177e4SLinus Torvalds } 14431da177e4SLinus Torvalds 14441da177e4SLinus Torvalds /* Replace a policy range. */ 14451da177e4SLinus Torvalds static int shared_policy_replace(struct shared_policy *sp, unsigned long start, 14461da177e4SLinus Torvalds unsigned long end, struct sp_node *new) 14471da177e4SLinus Torvalds { 14481da177e4SLinus Torvalds struct sp_node *n, *new2 = NULL; 14491da177e4SLinus Torvalds 14501da177e4SLinus Torvalds restart: 14511da177e4SLinus Torvalds spin_lock(&sp->lock); 14521da177e4SLinus Torvalds n = sp_lookup(sp, start, end); 14531da177e4SLinus Torvalds /* Take care of old policies in the same range. */ 14541da177e4SLinus Torvalds while (n && n->start < end) { 14551da177e4SLinus Torvalds struct rb_node *next = rb_next(&n->nd); 14561da177e4SLinus Torvalds if (n->start >= start) { 14571da177e4SLinus Torvalds if (n->end <= end) 14581da177e4SLinus Torvalds sp_delete(sp, n); 14591da177e4SLinus Torvalds else 14601da177e4SLinus Torvalds n->start = end; 14611da177e4SLinus Torvalds } else { 14621da177e4SLinus Torvalds /* Old policy spanning whole new range. */ 14631da177e4SLinus Torvalds if (n->end > end) { 14641da177e4SLinus Torvalds if (!new2) { 14651da177e4SLinus Torvalds spin_unlock(&sp->lock); 14661da177e4SLinus Torvalds new2 = sp_alloc(end, n->end, n->policy); 14671da177e4SLinus Torvalds if (!new2) 14681da177e4SLinus Torvalds return -ENOMEM; 14691da177e4SLinus Torvalds goto restart; 14701da177e4SLinus Torvalds } 14711da177e4SLinus Torvalds n->end = start; 14721da177e4SLinus Torvalds sp_insert(sp, new2); 14731da177e4SLinus Torvalds new2 = NULL; 14741da177e4SLinus Torvalds break; 14751da177e4SLinus Torvalds } else 14761da177e4SLinus Torvalds n->end = start; 14771da177e4SLinus Torvalds } 14781da177e4SLinus Torvalds if (!next) 14791da177e4SLinus Torvalds break; 14801da177e4SLinus Torvalds n = rb_entry(next, struct sp_node, nd); 14811da177e4SLinus Torvalds } 14821da177e4SLinus Torvalds if (new) 14831da177e4SLinus Torvalds sp_insert(sp, new); 14841da177e4SLinus Torvalds spin_unlock(&sp->lock); 14851da177e4SLinus Torvalds if (new2) { 14861da177e4SLinus Torvalds mpol_free(new2->policy); 14871da177e4SLinus Torvalds kmem_cache_free(sn_cache, new2); 14881da177e4SLinus Torvalds } 14891da177e4SLinus Torvalds return 0; 14901da177e4SLinus Torvalds } 14911da177e4SLinus Torvalds 14927339ff83SRobin Holt void mpol_shared_policy_init(struct shared_policy *info, int policy, 14937339ff83SRobin Holt nodemask_t *policy_nodes) 14947339ff83SRobin Holt { 14957339ff83SRobin Holt info->root = RB_ROOT; 14967339ff83SRobin Holt spin_lock_init(&info->lock); 14977339ff83SRobin Holt 14987339ff83SRobin Holt if (policy != MPOL_DEFAULT) { 14997339ff83SRobin Holt struct mempolicy *newpol; 15007339ff83SRobin Holt 15017339ff83SRobin Holt /* Falls back to MPOL_DEFAULT on any error */ 15027339ff83SRobin Holt newpol = mpol_new(policy, policy_nodes); 15037339ff83SRobin Holt if (!IS_ERR(newpol)) { 15047339ff83SRobin Holt /* Create pseudo-vma that contains just the policy */ 15057339ff83SRobin Holt struct vm_area_struct pvma; 15067339ff83SRobin Holt 15077339ff83SRobin Holt memset(&pvma, 0, sizeof(struct vm_area_struct)); 15087339ff83SRobin Holt /* Policy covers entire file */ 15097339ff83SRobin Holt pvma.vm_end = TASK_SIZE; 15107339ff83SRobin Holt mpol_set_shared_policy(info, &pvma, newpol); 15117339ff83SRobin Holt mpol_free(newpol); 15127339ff83SRobin Holt } 15137339ff83SRobin Holt } 15147339ff83SRobin Holt } 15157339ff83SRobin Holt 15161da177e4SLinus Torvalds int mpol_set_shared_policy(struct shared_policy *info, 15171da177e4SLinus Torvalds struct vm_area_struct *vma, struct mempolicy *npol) 15181da177e4SLinus Torvalds { 15191da177e4SLinus Torvalds int err; 15201da177e4SLinus Torvalds struct sp_node *new = NULL; 15211da177e4SLinus Torvalds unsigned long sz = vma_pages(vma); 15221da177e4SLinus Torvalds 15231da177e4SLinus Torvalds PDprintk("set_shared_policy %lx sz %lu %d %lx\n", 15241da177e4SLinus Torvalds vma->vm_pgoff, 15251da177e4SLinus Torvalds sz, npol? npol->policy : -1, 1526dfcd3c0dSAndi Kleen npol ? nodes_addr(npol->v.nodes)[0] : -1); 15271da177e4SLinus Torvalds 15281da177e4SLinus Torvalds if (npol) { 15291da177e4SLinus Torvalds new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); 15301da177e4SLinus Torvalds if (!new) 15311da177e4SLinus Torvalds return -ENOMEM; 15321da177e4SLinus Torvalds } 15331da177e4SLinus Torvalds err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); 15341da177e4SLinus Torvalds if (err && new) 15351da177e4SLinus Torvalds kmem_cache_free(sn_cache, new); 15361da177e4SLinus Torvalds return err; 15371da177e4SLinus Torvalds } 15381da177e4SLinus Torvalds 15391da177e4SLinus Torvalds /* Free a backing policy store on inode delete. */ 15401da177e4SLinus Torvalds void mpol_free_shared_policy(struct shared_policy *p) 15411da177e4SLinus Torvalds { 15421da177e4SLinus Torvalds struct sp_node *n; 15431da177e4SLinus Torvalds struct rb_node *next; 15441da177e4SLinus Torvalds 15451da177e4SLinus Torvalds if (!p->root.rb_node) 15461da177e4SLinus Torvalds return; 15471da177e4SLinus Torvalds spin_lock(&p->lock); 15481da177e4SLinus Torvalds next = rb_first(&p->root); 15491da177e4SLinus Torvalds while (next) { 15501da177e4SLinus Torvalds n = rb_entry(next, struct sp_node, nd); 15511da177e4SLinus Torvalds next = rb_next(&n->nd); 155290c5029eSAndi Kleen rb_erase(&n->nd, &p->root); 15531da177e4SLinus Torvalds mpol_free(n->policy); 15541da177e4SLinus Torvalds kmem_cache_free(sn_cache, n); 15551da177e4SLinus Torvalds } 15561da177e4SLinus Torvalds spin_unlock(&p->lock); 15571da177e4SLinus Torvalds } 15581da177e4SLinus Torvalds 15591da177e4SLinus Torvalds /* assumes fs == KERNEL_DS */ 15601da177e4SLinus Torvalds void __init numa_policy_init(void) 15611da177e4SLinus Torvalds { 15621da177e4SLinus Torvalds policy_cache = kmem_cache_create("numa_policy", 15631da177e4SLinus Torvalds sizeof(struct mempolicy), 15641da177e4SLinus Torvalds 0, SLAB_PANIC, NULL, NULL); 15651da177e4SLinus Torvalds 15661da177e4SLinus Torvalds sn_cache = kmem_cache_create("shared_policy_node", 15671da177e4SLinus Torvalds sizeof(struct sp_node), 15681da177e4SLinus Torvalds 0, SLAB_PANIC, NULL, NULL); 15691da177e4SLinus Torvalds 15701da177e4SLinus Torvalds /* Set interleaving policy for system init. This way not all 15711da177e4SLinus Torvalds the data structures allocated at system boot end up in node zero. */ 15721da177e4SLinus Torvalds 15738bccd85fSChristoph Lameter if (do_set_mempolicy(MPOL_INTERLEAVE, &node_online_map)) 15741da177e4SLinus Torvalds printk("numa_policy_init: interleaving failed\n"); 15751da177e4SLinus Torvalds } 15761da177e4SLinus Torvalds 15778bccd85fSChristoph Lameter /* Reset policy of current process to default */ 15781da177e4SLinus Torvalds void numa_default_policy(void) 15791da177e4SLinus Torvalds { 15808bccd85fSChristoph Lameter do_set_mempolicy(MPOL_DEFAULT, NULL); 15811da177e4SLinus Torvalds } 158268860ec1SPaul Jackson 158368860ec1SPaul Jackson /* Migrate a policy to a different set of nodes */ 158474cb2155SPaul Jackson void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask) 158568860ec1SPaul Jackson { 158674cb2155SPaul Jackson nodemask_t *mpolmask; 158768860ec1SPaul Jackson nodemask_t tmp; 158868860ec1SPaul Jackson 158968860ec1SPaul Jackson if (!pol) 159068860ec1SPaul Jackson return; 159174cb2155SPaul Jackson mpolmask = &pol->cpuset_mems_allowed; 159274cb2155SPaul Jackson if (nodes_equal(*mpolmask, *newmask)) 159374cb2155SPaul Jackson return; 159468860ec1SPaul Jackson 159568860ec1SPaul Jackson switch (pol->policy) { 159668860ec1SPaul Jackson case MPOL_DEFAULT: 159768860ec1SPaul Jackson break; 159868860ec1SPaul Jackson case MPOL_INTERLEAVE: 159974cb2155SPaul Jackson nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask); 160068860ec1SPaul Jackson pol->v.nodes = tmp; 160174cb2155SPaul Jackson *mpolmask = *newmask; 160274cb2155SPaul Jackson current->il_next = node_remap(current->il_next, 160374cb2155SPaul Jackson *mpolmask, *newmask); 160468860ec1SPaul Jackson break; 160568860ec1SPaul Jackson case MPOL_PREFERRED: 160668860ec1SPaul Jackson pol->v.preferred_node = node_remap(pol->v.preferred_node, 160774cb2155SPaul Jackson *mpolmask, *newmask); 160874cb2155SPaul Jackson *mpolmask = *newmask; 160968860ec1SPaul Jackson break; 161068860ec1SPaul Jackson case MPOL_BIND: { 161168860ec1SPaul Jackson nodemask_t nodes; 161268860ec1SPaul Jackson struct zone **z; 161368860ec1SPaul Jackson struct zonelist *zonelist; 161468860ec1SPaul Jackson 161568860ec1SPaul Jackson nodes_clear(nodes); 161668860ec1SPaul Jackson for (z = pol->v.zonelist->zones; *z; z++) 161768860ec1SPaul Jackson node_set((*z)->zone_pgdat->node_id, nodes); 161874cb2155SPaul Jackson nodes_remap(tmp, nodes, *mpolmask, *newmask); 161968860ec1SPaul Jackson nodes = tmp; 162068860ec1SPaul Jackson 162168860ec1SPaul Jackson zonelist = bind_zonelist(&nodes); 162268860ec1SPaul Jackson 162368860ec1SPaul Jackson /* If no mem, then zonelist is NULL and we keep old zonelist. 162468860ec1SPaul Jackson * If that old zonelist has no remaining mems_allowed nodes, 162568860ec1SPaul Jackson * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT. 162668860ec1SPaul Jackson */ 162768860ec1SPaul Jackson 162868860ec1SPaul Jackson if (zonelist) { 162968860ec1SPaul Jackson /* Good - got mem - substitute new zonelist */ 163068860ec1SPaul Jackson kfree(pol->v.zonelist); 163168860ec1SPaul Jackson pol->v.zonelist = zonelist; 163268860ec1SPaul Jackson } 163374cb2155SPaul Jackson *mpolmask = *newmask; 163468860ec1SPaul Jackson break; 163568860ec1SPaul Jackson } 163668860ec1SPaul Jackson default: 163768860ec1SPaul Jackson BUG(); 163868860ec1SPaul Jackson break; 163968860ec1SPaul Jackson } 164068860ec1SPaul Jackson } 164168860ec1SPaul Jackson 164268860ec1SPaul Jackson /* 164374cb2155SPaul Jackson * Wrapper for mpol_rebind_policy() that just requires task 164474cb2155SPaul Jackson * pointer, and updates task mempolicy. 164568860ec1SPaul Jackson */ 164674cb2155SPaul Jackson 164774cb2155SPaul Jackson void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) 164868860ec1SPaul Jackson { 164974cb2155SPaul Jackson mpol_rebind_policy(tsk->mempolicy, new); 165068860ec1SPaul Jackson } 16511a75a6c8SChristoph Lameter 16521a75a6c8SChristoph Lameter /* 16534225399aSPaul Jackson * Rebind each vma in mm to new nodemask. 16544225399aSPaul Jackson * 16554225399aSPaul Jackson * Call holding a reference to mm. Takes mm->mmap_sem during call. 16564225399aSPaul Jackson */ 16574225399aSPaul Jackson 16584225399aSPaul Jackson void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) 16594225399aSPaul Jackson { 16604225399aSPaul Jackson struct vm_area_struct *vma; 16614225399aSPaul Jackson 16624225399aSPaul Jackson down_write(&mm->mmap_sem); 16634225399aSPaul Jackson for (vma = mm->mmap; vma; vma = vma->vm_next) 16644225399aSPaul Jackson mpol_rebind_policy(vma->vm_policy, new); 16654225399aSPaul Jackson up_write(&mm->mmap_sem); 16664225399aSPaul Jackson } 16674225399aSPaul Jackson 16684225399aSPaul Jackson /* 16691a75a6c8SChristoph Lameter * Display pages allocated per node and memory policy via /proc. 16701a75a6c8SChristoph Lameter */ 16711a75a6c8SChristoph Lameter 16721a75a6c8SChristoph Lameter static const char *policy_types[] = { "default", "prefer", "bind", 16731a75a6c8SChristoph Lameter "interleave" }; 16741a75a6c8SChristoph Lameter 16751a75a6c8SChristoph Lameter /* 16761a75a6c8SChristoph Lameter * Convert a mempolicy into a string. 16771a75a6c8SChristoph Lameter * Returns the number of characters in buffer (if positive) 16781a75a6c8SChristoph Lameter * or an error (negative) 16791a75a6c8SChristoph Lameter */ 16801a75a6c8SChristoph Lameter static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) 16811a75a6c8SChristoph Lameter { 16821a75a6c8SChristoph Lameter char *p = buffer; 16831a75a6c8SChristoph Lameter int l; 16841a75a6c8SChristoph Lameter nodemask_t nodes; 16851a75a6c8SChristoph Lameter int mode = pol ? pol->policy : MPOL_DEFAULT; 16861a75a6c8SChristoph Lameter 16871a75a6c8SChristoph Lameter switch (mode) { 16881a75a6c8SChristoph Lameter case MPOL_DEFAULT: 16891a75a6c8SChristoph Lameter nodes_clear(nodes); 16901a75a6c8SChristoph Lameter break; 16911a75a6c8SChristoph Lameter 16921a75a6c8SChristoph Lameter case MPOL_PREFERRED: 16931a75a6c8SChristoph Lameter nodes_clear(nodes); 16941a75a6c8SChristoph Lameter node_set(pol->v.preferred_node, nodes); 16951a75a6c8SChristoph Lameter break; 16961a75a6c8SChristoph Lameter 16971a75a6c8SChristoph Lameter case MPOL_BIND: 16981a75a6c8SChristoph Lameter get_zonemask(pol, &nodes); 16991a75a6c8SChristoph Lameter break; 17001a75a6c8SChristoph Lameter 17011a75a6c8SChristoph Lameter case MPOL_INTERLEAVE: 17021a75a6c8SChristoph Lameter nodes = pol->v.nodes; 17031a75a6c8SChristoph Lameter break; 17041a75a6c8SChristoph Lameter 17051a75a6c8SChristoph Lameter default: 17061a75a6c8SChristoph Lameter BUG(); 17071a75a6c8SChristoph Lameter return -EFAULT; 17081a75a6c8SChristoph Lameter } 17091a75a6c8SChristoph Lameter 17101a75a6c8SChristoph Lameter l = strlen(policy_types[mode]); 17111a75a6c8SChristoph Lameter if (buffer + maxlen < p + l + 1) 17121a75a6c8SChristoph Lameter return -ENOSPC; 17131a75a6c8SChristoph Lameter 17141a75a6c8SChristoph Lameter strcpy(p, policy_types[mode]); 17151a75a6c8SChristoph Lameter p += l; 17161a75a6c8SChristoph Lameter 17171a75a6c8SChristoph Lameter if (!nodes_empty(nodes)) { 17181a75a6c8SChristoph Lameter if (buffer + maxlen < p + 2) 17191a75a6c8SChristoph Lameter return -ENOSPC; 17201a75a6c8SChristoph Lameter *p++ = '='; 17211a75a6c8SChristoph Lameter p += nodelist_scnprintf(p, buffer + maxlen - p, nodes); 17221a75a6c8SChristoph Lameter } 17231a75a6c8SChristoph Lameter return p - buffer; 17241a75a6c8SChristoph Lameter } 17251a75a6c8SChristoph Lameter 17261a75a6c8SChristoph Lameter struct numa_maps { 17271a75a6c8SChristoph Lameter unsigned long pages; 17281a75a6c8SChristoph Lameter unsigned long anon; 17291a75a6c8SChristoph Lameter unsigned long mapped; 17301a75a6c8SChristoph Lameter unsigned long mapcount_max; 17311a75a6c8SChristoph Lameter unsigned long node[MAX_NUMNODES]; 17321a75a6c8SChristoph Lameter }; 17331a75a6c8SChristoph Lameter 17341a75a6c8SChristoph Lameter static void gather_stats(struct page *page, void *private) 17351a75a6c8SChristoph Lameter { 17361a75a6c8SChristoph Lameter struct numa_maps *md = private; 17371a75a6c8SChristoph Lameter int count = page_mapcount(page); 17381a75a6c8SChristoph Lameter 17391a75a6c8SChristoph Lameter if (count) 17401a75a6c8SChristoph Lameter md->mapped++; 17411a75a6c8SChristoph Lameter 17421a75a6c8SChristoph Lameter if (count > md->mapcount_max) 17431a75a6c8SChristoph Lameter md->mapcount_max = count; 17441a75a6c8SChristoph Lameter 17451a75a6c8SChristoph Lameter md->pages++; 17461a75a6c8SChristoph Lameter 17471a75a6c8SChristoph Lameter if (PageAnon(page)) 17481a75a6c8SChristoph Lameter md->anon++; 17491a75a6c8SChristoph Lameter 17501a75a6c8SChristoph Lameter md->node[page_to_nid(page)]++; 17511a75a6c8SChristoph Lameter cond_resched(); 17521a75a6c8SChristoph Lameter } 17531a75a6c8SChristoph Lameter 17541a75a6c8SChristoph Lameter int show_numa_map(struct seq_file *m, void *v) 17551a75a6c8SChristoph Lameter { 17561a75a6c8SChristoph Lameter struct task_struct *task = m->private; 17571a75a6c8SChristoph Lameter struct vm_area_struct *vma = v; 17581a75a6c8SChristoph Lameter struct numa_maps *md; 17591a75a6c8SChristoph Lameter int n; 17601a75a6c8SChristoph Lameter char buffer[50]; 17611a75a6c8SChristoph Lameter 17621a75a6c8SChristoph Lameter if (!vma->vm_mm) 17631a75a6c8SChristoph Lameter return 0; 17641a75a6c8SChristoph Lameter 17651a75a6c8SChristoph Lameter md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL); 17661a75a6c8SChristoph Lameter if (!md) 17671a75a6c8SChristoph Lameter return 0; 17681a75a6c8SChristoph Lameter 17691a75a6c8SChristoph Lameter check_pgd_range(vma, vma->vm_start, vma->vm_end, 17701a75a6c8SChristoph Lameter &node_online_map, MPOL_MF_STATS, md); 17711a75a6c8SChristoph Lameter 17721a75a6c8SChristoph Lameter if (md->pages) { 17731a75a6c8SChristoph Lameter mpol_to_str(buffer, sizeof(buffer), 17741a75a6c8SChristoph Lameter get_vma_policy(task, vma, vma->vm_start)); 17751a75a6c8SChristoph Lameter 17761a75a6c8SChristoph Lameter seq_printf(m, "%08lx %s pages=%lu mapped=%lu maxref=%lu", 17771a75a6c8SChristoph Lameter vma->vm_start, buffer, md->pages, 17781a75a6c8SChristoph Lameter md->mapped, md->mapcount_max); 17791a75a6c8SChristoph Lameter 17801a75a6c8SChristoph Lameter if (md->anon) 17811a75a6c8SChristoph Lameter seq_printf(m," anon=%lu",md->anon); 17821a75a6c8SChristoph Lameter 17831a75a6c8SChristoph Lameter for_each_online_node(n) 17841a75a6c8SChristoph Lameter if (md->node[n]) 17851a75a6c8SChristoph Lameter seq_printf(m, " N%d=%lu", n, md->node[n]); 17861a75a6c8SChristoph Lameter 17871a75a6c8SChristoph Lameter seq_putc(m, '\n'); 17881a75a6c8SChristoph Lameter } 17891a75a6c8SChristoph Lameter kfree(md); 17901a75a6c8SChristoph Lameter 17911a75a6c8SChristoph Lameter if (m->count < m->size) 17921a75a6c8SChristoph Lameter m->version = (vma != get_gate_vma(task)) ? vma->vm_start : 0; 17931a75a6c8SChristoph Lameter return 0; 17941a75a6c8SChristoph Lameter } 17951a75a6c8SChristoph Lameter 1796