1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared 5 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations. 6 */ 7 8 #include "vma_internal.h" 9 #include "vma.h" 10 11 /* SLAB cache for vm_area_struct structures */ 12 static struct kmem_cache *vm_area_cachep; 13 14 void __init vma_state_init(void) 15 { 16 struct kmem_cache_args args = { 17 .use_freeptr_offset = true, 18 .freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr), 19 .sheaf_capacity = 32, 20 }; 21 22 vm_area_cachep = kmem_cache_create("vm_area_struct", 23 sizeof(struct vm_area_struct), &args, 24 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| 25 SLAB_ACCOUNT); 26 } 27 28 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) 29 { 30 struct vm_area_struct *vma; 31 32 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 33 if (!vma) 34 return NULL; 35 36 vma_init(vma, mm); 37 38 return vma; 39 } 40 41 static void vm_area_init_from(const struct vm_area_struct *src, 42 struct vm_area_struct *dest) 43 { 44 dest->vm_mm = src->vm_mm; 45 dest->vm_ops = src->vm_ops; 46 dest->vm_start = src->vm_start; 47 dest->vm_end = src->vm_end; 48 dest->anon_vma = src->anon_vma; 49 dest->vm_pgoff = src->vm_pgoff; 50 dest->vm_file = src->vm_file; 51 dest->vm_private_data = src->vm_private_data; 52 vm_flags_init(dest, src->vm_flags); 53 memcpy(&dest->vm_page_prot, &src->vm_page_prot, 54 sizeof(dest->vm_page_prot)); 55 /* 56 * src->shared.rb may be modified concurrently when called from 57 * dup_mmap(), but the clone will reinitialize it. 58 */ 59 data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared))); 60 memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx, 61 sizeof(dest->vm_userfaultfd_ctx)); 62 #ifdef CONFIG_ANON_VMA_NAME 63 dest->anon_name = src->anon_name; 64 #endif 65 #ifdef CONFIG_SWAP 66 memcpy(&dest->swap_readahead_info, &src->swap_readahead_info, 67 sizeof(dest->swap_readahead_info)); 68 #endif 69 #ifndef CONFIG_MMU 70 dest->vm_region = src->vm_region; 71 #endif 72 #ifdef CONFIG_NUMA 73 dest->vm_policy = src->vm_policy; 74 #endif 75 #ifdef __HAVE_PFNMAP_TRACKING 76 dest->pfnmap_track_ctx = NULL; 77 #endif 78 } 79 80 #ifdef __HAVE_PFNMAP_TRACKING 81 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig, 82 struct vm_area_struct *new) 83 { 84 struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx; 85 86 if (likely(!ctx)) 87 return 0; 88 89 /* 90 * We don't expect to ever hit this. If ever required, we would have 91 * to duplicate the tracking. 92 */ 93 if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX)) 94 return -ENOMEM; 95 kref_get(&ctx->kref); 96 new->pfnmap_track_ctx = ctx; 97 return 0; 98 } 99 100 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma) 101 { 102 struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx; 103 104 if (likely(!ctx)) 105 return; 106 107 kref_put(&ctx->kref, pfnmap_track_ctx_release); 108 vma->pfnmap_track_ctx = NULL; 109 } 110 #else 111 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig, 112 struct vm_area_struct *new) 113 { 114 return 0; 115 } 116 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma) 117 { 118 } 119 #endif 120 121 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) 122 { 123 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 124 125 if (!new) 126 return NULL; 127 128 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); 129 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); 130 vm_area_init_from(orig, new); 131 132 if (vma_pfnmap_track_ctx_dup(orig, new)) { 133 kmem_cache_free(vm_area_cachep, new); 134 return NULL; 135 } 136 vma_lock_init(new, true); 137 INIT_LIST_HEAD(&new->anon_vma_chain); 138 vma_numab_state_init(new); 139 dup_anon_vma_name(orig, new); 140 141 return new; 142 } 143 144 void vm_area_free(struct vm_area_struct *vma) 145 { 146 /* The vma should be detached while being destroyed. */ 147 vma_assert_detached(vma); 148 vma_numab_state_free(vma); 149 free_anon_vma_name(vma); 150 vma_pfnmap_track_ctx_release(vma); 151 kmem_cache_free(vm_area_cachep, vma); 152 } 153