1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared 5 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations. 6 */ 7 8 /* 9 * To allow for userland testing we place internal dependencies in 10 * vma_internal.h and external VMA API declarations in vma.h. 11 */ 12 #include "vma_internal.h" 13 #include "vma.h" 14 15 /* SLAB cache for vm_area_struct structures */ 16 static struct kmem_cache *vm_area_cachep; 17 18 void __init vma_state_init(void) 19 { 20 struct kmem_cache_args args = { 21 .use_freeptr_offset = true, 22 .freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr), 23 .sheaf_capacity = 32, 24 }; 25 26 vm_area_cachep = kmem_cache_create("vm_area_struct", 27 sizeof(struct vm_area_struct), &args, 28 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| 29 SLAB_ACCOUNT); 30 } 31 32 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) 33 { 34 struct vm_area_struct *vma; 35 36 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 37 if (!vma) 38 return NULL; 39 40 vma_init(vma, mm); 41 42 return vma; 43 } 44 45 static void vm_area_init_from(const struct vm_area_struct *src, 46 struct vm_area_struct *dest) 47 { 48 dest->vm_mm = src->vm_mm; 49 dest->vm_ops = src->vm_ops; 50 dest->vm_start = src->vm_start; 51 dest->vm_end = src->vm_end; 52 dest->anon_vma = src->anon_vma; 53 dest->vm_pgoff = vma_start_pgoff(src); 54 __vma_set_anon_pgoff(dest, vma_start_anon_pgoff(src)); 55 dest->vm_file = src->vm_file; 56 dest->vm_private_data = src->vm_private_data; 57 vm_flags_init(dest, src->vm_flags); 58 memcpy(&dest->vm_page_prot, &src->vm_page_prot, 59 sizeof(dest->vm_page_prot)); 60 /* 61 * src->shared.rb may be modified concurrently when called from 62 * dup_mmap(), but the clone will reinitialize it. 63 */ 64 data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared))); 65 memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx, 66 sizeof(dest->vm_userfaultfd_ctx)); 67 #ifdef CONFIG_ANON_VMA_NAME 68 dest->anon_name = src->anon_name; 69 #endif 70 #ifdef CONFIG_SWAP 71 memcpy(&dest->swap_readahead_info, &src->swap_readahead_info, 72 sizeof(dest->swap_readahead_info)); 73 #endif 74 #ifndef CONFIG_MMU 75 dest->vm_region = src->vm_region; 76 #endif 77 #ifdef CONFIG_NUMA 78 dest->vm_policy = src->vm_policy; 79 #endif 80 #ifdef __HAVE_PFNMAP_TRACKING 81 dest->pfnmap_track_ctx = NULL; 82 #endif 83 } 84 85 #ifdef __HAVE_PFNMAP_TRACKING 86 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig, 87 struct vm_area_struct *new) 88 { 89 struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx; 90 91 if (likely(!ctx)) 92 return 0; 93 94 /* 95 * We don't expect to ever hit this. If ever required, we would have 96 * to duplicate the tracking. 97 */ 98 if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX)) 99 return -ENOMEM; 100 kref_get(&ctx->kref); 101 new->pfnmap_track_ctx = ctx; 102 return 0; 103 } 104 105 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma) 106 { 107 struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx; 108 109 if (likely(!ctx)) 110 return; 111 112 kref_put(&ctx->kref, pfnmap_track_ctx_release); 113 vma->pfnmap_track_ctx = NULL; 114 } 115 #else 116 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig, 117 struct vm_area_struct *new) 118 { 119 return 0; 120 } 121 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma) 122 { 123 } 124 #endif 125 126 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) 127 { 128 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 129 130 if (!new) 131 return NULL; 132 133 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); 134 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); 135 vm_area_init_from(orig, new); 136 137 if (vma_pfnmap_track_ctx_dup(orig, new)) { 138 kmem_cache_free(vm_area_cachep, new); 139 return NULL; 140 } 141 vma_lock_init(new, true); 142 INIT_LIST_HEAD(&new->anon_vma_chain); 143 vma_numab_state_init(new); 144 dup_anon_vma_name(orig, new); 145 146 return new; 147 } 148 149 void vm_area_free(struct vm_area_struct *vma) 150 { 151 /* The vma should be detached while being destroyed. */ 152 vma_assert_detached(vma); 153 vma_numab_state_free(vma); 154 free_anon_vma_name(vma); 155 vma_pfnmap_track_ctx_release(vma); 156 kmem_cache_free(vm_area_cachep, vma); 157 } 158