1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared 5 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations. 6 */ 7 8 /* 9 * To allow for userland testing we place internal dependencies in 10 * vma_internal.h and external VMA API declarations in vma.h. 11 */ 12 #include "vma_internal.h" 13 #include "vma.h" 14 15 /* SLAB cache for vm_area_struct structures */ 16 static struct kmem_cache *vm_area_cachep; 17 18 void __init vma_state_init(void) 19 { 20 struct kmem_cache_args args = { 21 .use_freeptr_offset = true, 22 .freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr), 23 .sheaf_capacity = 32, 24 }; 25 26 vm_area_cachep = kmem_cache_create("vm_area_struct", 27 sizeof(struct vm_area_struct), &args, 28 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| 29 SLAB_ACCOUNT); 30 } 31 32 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) 33 { 34 struct vm_area_struct *vma; 35 36 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 37 if (!vma) 38 return NULL; 39 40 vma_init(vma, mm); 41 42 return vma; 43 } 44 45 static void vm_area_init_from(const struct vm_area_struct *src, 46 struct vm_area_struct *dest) 47 { 48 dest->vm_mm = src->vm_mm; 49 dest->vm_ops = src->vm_ops; 50 dest->vm_start = src->vm_start; 51 dest->vm_end = src->vm_end; 52 dest->anon_vma = src->anon_vma; 53 dest->vm_pgoff = vma_start_pgoff(src); 54 dest->vm_file = src->vm_file; 55 dest->vm_private_data = src->vm_private_data; 56 vm_flags_init(dest, src->vm_flags); 57 memcpy(&dest->vm_page_prot, &src->vm_page_prot, 58 sizeof(dest->vm_page_prot)); 59 /* 60 * src->shared.rb may be modified concurrently when called from 61 * dup_mmap(), but the clone will reinitialize it. 62 */ 63 data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared))); 64 memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx, 65 sizeof(dest->vm_userfaultfd_ctx)); 66 #ifdef CONFIG_ANON_VMA_NAME 67 dest->anon_name = src->anon_name; 68 #endif 69 #ifdef CONFIG_SWAP 70 memcpy(&dest->swap_readahead_info, &src->swap_readahead_info, 71 sizeof(dest->swap_readahead_info)); 72 #endif 73 #ifndef CONFIG_MMU 74 dest->vm_region = src->vm_region; 75 #endif 76 #ifdef CONFIG_NUMA 77 dest->vm_policy = src->vm_policy; 78 #endif 79 #ifdef __HAVE_PFNMAP_TRACKING 80 dest->pfnmap_track_ctx = NULL; 81 #endif 82 } 83 84 #ifdef __HAVE_PFNMAP_TRACKING 85 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig, 86 struct vm_area_struct *new) 87 { 88 struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx; 89 90 if (likely(!ctx)) 91 return 0; 92 93 /* 94 * We don't expect to ever hit this. If ever required, we would have 95 * to duplicate the tracking. 96 */ 97 if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX)) 98 return -ENOMEM; 99 kref_get(&ctx->kref); 100 new->pfnmap_track_ctx = ctx; 101 return 0; 102 } 103 104 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma) 105 { 106 struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx; 107 108 if (likely(!ctx)) 109 return; 110 111 kref_put(&ctx->kref, pfnmap_track_ctx_release); 112 vma->pfnmap_track_ctx = NULL; 113 } 114 #else 115 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig, 116 struct vm_area_struct *new) 117 { 118 return 0; 119 } 120 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma) 121 { 122 } 123 #endif 124 125 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) 126 { 127 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 128 129 if (!new) 130 return NULL; 131 132 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); 133 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); 134 vm_area_init_from(orig, new); 135 136 if (vma_pfnmap_track_ctx_dup(orig, new)) { 137 kmem_cache_free(vm_area_cachep, new); 138 return NULL; 139 } 140 vma_lock_init(new, true); 141 INIT_LIST_HEAD(&new->anon_vma_chain); 142 vma_numab_state_init(new); 143 dup_anon_vma_name(orig, new); 144 145 return new; 146 } 147 148 void vm_area_free(struct vm_area_struct *vma) 149 { 150 /* The vma should be detached while being destroyed. */ 151 vma_assert_detached(vma); 152 vma_numab_state_free(vma); 153 free_anon_vma_name(vma); 154 vma_pfnmap_track_ctx_release(vma); 155 kmem_cache_free(vm_area_cachep, vma); 156 } 157