xref: /linux/mm/secretmem.c (revision 3a2c4d55e32ad65efebdb6de44eef3bfa08bb49d) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corporation, 2021
4  *
5  * Author: Mike Rapoport <rppt@linux.ibm.com>
6  */
7 
8 #include <linux/mm.h>
9 #include <linux/fs.h>
10 #include <linux/swap.h>
11 #include <linux/mount.h>
12 #include <linux/memfd.h>
13 #include <linux/bitops.h>
14 #include <linux/printk.h>
15 #include <linux/pagemap.h>
16 #include <linux/syscalls.h>
17 #include <linux/pseudo_fs.h>
18 #include <linux/secretmem.h>
19 #include <linux/set_memory.h>
20 #include <linux/sched/signal.h>
21 #include <linux/sched/user.h>
22 #include <linux/cred.h>
23 
24 #include <uapi/linux/magic.h>
25 
26 #include <asm/tlbflush.h>
27 
28 #include "internal.h"
29 
30 #undef pr_fmt
31 #define pr_fmt(fmt) "secretmem: " fmt
32 
33 /*
34  * Define mode and flag masks to allow validation of the system call
35  * parameters.
36  */
37 #define SECRETMEM_MODE_MASK	(0x0)
38 #define SECRETMEM_FLAGS_MASK	SECRETMEM_MODE_MASK
39 
40 static bool secretmem_enable __ro_after_init = 1;
41 module_param_named(enable, secretmem_enable, bool, 0400);
42 MODULE_PARM_DESC(secretmem_enable,
43 		 "Enable secretmem and memfd_secret(2) system call");
44 
45 static atomic_t secretmem_users;
46 
47 bool secretmem_active(void)
48 {
49 	return !!atomic_read(&secretmem_users);
50 }
51 
52 struct secretmem_inode_state {
53 	struct user_struct	*user;
54 	atomic_long_t		nr_pages_accounted;
55 };
56 
57 static bool __secretmem_account_pages(struct user_struct *user,
58 		unsigned long nr_pages)
59 {
60 	unsigned long page_limit, cur_pages, new_pages;
61 
62 	if (!nr_pages)
63 		return true;
64 
65 	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
66 
67 	cur_pages = atomic_long_read(&user->locked_vm);
68 	do {
69 		new_pages = cur_pages + nr_pages;
70 		if (new_pages > page_limit)
71 			return false;
72 	} while (!atomic_long_try_cmpxchg(&user->locked_vm,
73 					  &cur_pages, new_pages));
74 	return true;
75 }
76 
77 static bool secretmem_account_folio(struct secretmem_inode_state *state,
78 		const struct folio *folio)
79 {
80 	const unsigned long nr_pages = folio_nr_pages(folio);
81 
82 	if (!__secretmem_account_pages(state->user, nr_pages))
83 		return false;
84 
85 	atomic_long_add(nr_pages, &state->nr_pages_accounted);
86 	return true;
87 }
88 
89 static void __secretmem_unaccount_pages(struct secretmem_inode_state *state,
90 		unsigned long nr_pages)
91 {
92 	atomic_long_sub(nr_pages, &state->user->locked_vm);
93 	atomic_long_sub(nr_pages, &state->nr_pages_accounted);
94 }
95 
96 static void secretmem_unaccount_folio(struct secretmem_inode_state *state,
97 		struct folio *folio)
98 {
99 	__secretmem_unaccount_pages(state, folio_nr_pages(folio));
100 }
101 
102 static void secretmem_unaccount_all_folios(struct secretmem_inode_state *state)
103 {
104 	const unsigned long nr_pages_accounted =
105 		atomic_long_read(&state->nr_pages_accounted);
106 
107 	__secretmem_unaccount_pages(state, nr_pages_accounted);
108 }
109 
110 static vm_fault_t secretmem_fault(struct vm_fault *vmf)
111 {
112 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
113 	struct inode *inode = file_inode(vmf->vma->vm_file);
114 	struct secretmem_inode_state *state = inode->i_private;
115 	pgoff_t offset = vmf->pgoff;
116 	gfp_t gfp = vmf->gfp_mask;
117 	unsigned long addr;
118 	struct folio *folio;
119 	vm_fault_t ret;
120 	int err;
121 
122 	if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode))
123 		return vmf_error(-EINVAL);
124 
125 	filemap_invalidate_lock_shared(mapping);
126 
127 retry:
128 	folio = filemap_lock_folio(mapping, offset);
129 	if (IS_ERR(folio)) {
130 		folio = folio_alloc(gfp | __GFP_ZERO, 0);
131 		if (!folio) {
132 			ret = VM_FAULT_OOM;
133 			goto out;
134 		}
135 
136 		if (!secretmem_account_folio(state, folio)) {
137 			folio_put(folio);
138 			ret = VM_FAULT_SIGBUS;
139 			goto out;
140 		}
141 
142 		err = set_direct_map_invalid_noflush(folio_page(folio, 0));
143 		if (err) {
144 			secretmem_unaccount_folio(state, folio);
145 			folio_put(folio);
146 			ret = vmf_error(err);
147 			goto out;
148 		}
149 
150 		__folio_mark_uptodate(folio);
151 		err = filemap_add_folio(mapping, folio, offset, gfp);
152 		if (unlikely(err)) {
153 			secretmem_unaccount_folio(state, folio);
154 			/*
155 			 * If a split of large page was required, it
156 			 * already happened when we marked the page invalid
157 			 * which guarantees that this call won't fail
158 			 */
159 			set_direct_map_default_noflush(folio_page(folio, 0));
160 			folio_put(folio);
161 			if (err == -EEXIST)
162 				goto retry;
163 
164 			ret = vmf_error(err);
165 			goto out;
166 		}
167 
168 		addr = (unsigned long)folio_address(folio);
169 		flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
170 	}
171 
172 	vmf->page = folio_file_page(folio, vmf->pgoff);
173 	ret = VM_FAULT_LOCKED;
174 
175 out:
176 	filemap_invalidate_unlock_shared(mapping);
177 	return ret;
178 }
179 
180 static const struct vm_operations_struct secretmem_vm_ops = {
181 	.fault = secretmem_fault,
182 };
183 
184 static void secretmem_destroy_inode_priv(struct inode *inode)
185 {
186 	struct secretmem_inode_state *state = inode->i_private;
187 
188 	secretmem_unaccount_all_folios(state);
189 	free_uid(state->user);
190 	kfree(state);
191 	inode->i_private = NULL;
192 }
193 
194 static int secretmem_release(struct inode *inode, struct file *file)
195 {
196 	atomic_dec(&secretmem_users);
197 	secretmem_destroy_inode_priv(inode);
198 
199 	return 0;
200 }
201 
202 static int secretmem_mmap_prepare(struct vm_area_desc *desc)
203 {
204 	if (!vma_desc_test_any(desc, VMA_SHARED_BIT, VMA_MAYSHARE_BIT))
205 		return -EINVAL;
206 
207 	vma_desc_set_flags(desc, VMA_DONTDUMP_BIT);
208 	desc->vm_ops = &secretmem_vm_ops;
209 
210 	return 0;
211 }
212 
213 bool vma_is_secretmem(struct vm_area_struct *vma)
214 {
215 	return vma->vm_ops == &secretmem_vm_ops;
216 }
217 
218 static const struct file_operations secretmem_fops = {
219 	.release	= secretmem_release,
220 	.mmap_prepare	= secretmem_mmap_prepare,
221 };
222 
223 static int secretmem_migrate_folio(struct address_space *mapping,
224 		struct folio *dst, struct folio *src, enum migrate_mode mode)
225 {
226 	return -EBUSY;
227 }
228 
229 static void secretmem_free_folio(struct folio *folio)
230 {
231 	set_direct_map_default_noflush(folio_page(folio, 0));
232 	folio_zero_segment(folio, 0, folio_size(folio));
233 }
234 
235 const struct address_space_operations secretmem_aops = {
236 	.dirty_folio	= noop_dirty_folio,
237 	.free_folio	= secretmem_free_folio,
238 	.migrate_folio	= secretmem_migrate_folio,
239 };
240 
241 static int secretmem_setattr(struct mnt_idmap *idmap,
242 			     struct dentry *dentry, struct iattr *iattr)
243 {
244 	struct inode *inode = d_inode(dentry);
245 	struct address_space *mapping = inode->i_mapping;
246 	unsigned int ia_valid = iattr->ia_valid;
247 	int ret;
248 
249 	filemap_invalidate_lock(mapping);
250 
251 	if ((ia_valid & ATTR_SIZE) && inode->i_size)
252 		ret = -EINVAL;
253 	else
254 		ret = simple_setattr(idmap, dentry, iattr);
255 
256 	filemap_invalidate_unlock(mapping);
257 
258 	return ret;
259 }
260 
261 static const struct inode_operations secretmem_iops = {
262 	.setattr = secretmem_setattr,
263 };
264 
265 static struct vfsmount *secretmem_mnt;
266 
267 static int secretmem_init_inode_priv(struct inode *inode)
268 {
269 	struct secretmem_inode_state *state;
270 
271 	state = kzalloc_obj(*state);
272 	if (!state)
273 		return -ENOMEM;
274 
275 	state->user = get_uid(current_user());
276 	inode->i_private = state;
277 	return 0;
278 }
279 
280 static struct file *secretmem_file_create(unsigned long flags)
281 {
282 	struct file *file;
283 	struct inode *inode;
284 	const char *anon_name = "[secretmem]";
285 	int err;
286 
287 	inode = anon_inode_make_secure_inode(secretmem_mnt->mnt_sb, anon_name, NULL);
288 	if (IS_ERR(inode))
289 		return ERR_CAST(inode);
290 
291 	err = secretmem_init_inode_priv(inode);
292 	if (err)
293 		goto err_free_inode;
294 
295 	file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem",
296 				 O_RDWR | O_LARGEFILE, &secretmem_fops);
297 	if (IS_ERR(file)) {
298 		err = PTR_ERR(file);
299 		goto err_free_priv;
300 	}
301 
302 	mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
303 	mapping_set_unevictable(inode->i_mapping);
304 
305 	inode->i_op = &secretmem_iops;
306 	inode->i_mapping->a_ops = &secretmem_aops;
307 
308 	/* pretend we are a normal file with zero size */
309 	inode->i_mode |= S_IFREG;
310 	inode->i_size = 0;
311 
312 	atomic_inc(&secretmem_users);
313 
314 	return file;
315 err_free_priv:
316 	secretmem_destroy_inode_priv(inode);
317 err_free_inode:
318 	iput(inode);
319 	return ERR_PTR(err);
320 }
321 
322 SYSCALL_DEFINE1(memfd_secret, unsigned int, flags)
323 {
324 	/* make sure local flags do not conflict with global fcntl.h */
325 	BUILD_BUG_ON(SECRETMEM_FLAGS_MASK & O_CLOEXEC);
326 
327 	if (!secretmem_enable || !can_set_direct_map())
328 		return -ENOSYS;
329 
330 	if (flags & ~(SECRETMEM_FLAGS_MASK | O_CLOEXEC))
331 		return -EINVAL;
332 	if (atomic_read(&secretmem_users) < 0)
333 		return -ENFILE;
334 
335 	return FD_ADD(flags & O_CLOEXEC, secretmem_file_create(flags));
336 }
337 
338 static int secretmem_init_fs_context(struct fs_context *fc)
339 {
340 	struct pseudo_fs_context *ctx;
341 
342 	ctx = init_pseudo(fc, SECRETMEM_MAGIC);
343 	if (!ctx)
344 		return -ENOMEM;
345 
346 	return 0;
347 }
348 
349 static struct file_system_type secretmem_fs = {
350 	.name		= "secretmem",
351 	.init_fs_context = secretmem_init_fs_context,
352 	.kill_sb	= kill_anon_super,
353 };
354 
355 static int __init secretmem_init(void)
356 {
357 	if (!secretmem_enable || !can_set_direct_map())
358 		return 0;
359 
360 	secretmem_mnt = kern_mount(&secretmem_fs);
361 	if (IS_ERR(secretmem_mnt))
362 		return PTR_ERR(secretmem_mnt);
363 
364 	return 0;
365 }
366 fs_initcall(secretmem_init);
367