1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * This contains encryption functions for per-file encryption. 4 * 5 * Copyright (C) 2015, Google, Inc. 6 * Copyright (C) 2015, Motorola Mobility 7 * 8 * Written by Michael Halcrow, 2014. 9 * 10 * Filename encryption additions 11 * Uday Savagaonkar, 2014 12 * Encryption policy handling additions 13 * Ildar Muslukhov, 2014 14 * Add fscrypt_pullback_bio_page() 15 * Jaegeuk Kim, 2015. 16 * 17 * This has not yet undergone a rigorous security audit. 18 * 19 * The usage of AES-XTS should conform to recommendations in NIST 20 * Special Publication 800-38E and IEEE P1619/D16. 21 */ 22 23 #include <linux/pagemap.h> 24 #include <linux/mempool.h> 25 #include <linux/module.h> 26 #include <linux/scatterlist.h> 27 #include <linux/ratelimit.h> 28 #include <linux/dcache.h> 29 #include <linux/namei.h> 30 #include <crypto/skcipher.h> 31 #include "fscrypt_private.h" 32 33 static unsigned int num_prealloc_crypto_pages = 32; 34 35 module_param(num_prealloc_crypto_pages, uint, 0444); 36 MODULE_PARM_DESC(num_prealloc_crypto_pages, 37 "Number of crypto pages to preallocate"); 38 39 static mempool_t *fscrypt_bounce_page_pool = NULL; 40 41 static struct workqueue_struct *fscrypt_read_workqueue; 42 static DEFINE_MUTEX(fscrypt_init_mutex); 43 44 struct kmem_cache *fscrypt_info_cachep; 45 46 void fscrypt_enqueue_decrypt_work(struct work_struct *work) 47 { 48 queue_work(fscrypt_read_workqueue, work); 49 } 50 EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); 51 52 struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) 53 { 54 return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); 55 } 56 57 /** 58 * fscrypt_free_bounce_page() - free a ciphertext bounce page 59 * 60 * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(), 61 * or by fscrypt_alloc_bounce_page() directly. 62 */ 63 void fscrypt_free_bounce_page(struct page *bounce_page) 64 { 65 if (!bounce_page) 66 return; 67 set_page_private(bounce_page, (unsigned long)NULL); 68 ClearPagePrivate(bounce_page); 69 mempool_free(bounce_page, fscrypt_bounce_page_pool); 70 } 71 EXPORT_SYMBOL(fscrypt_free_bounce_page); 72 73 void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num, 74 const struct fscrypt_info *ci) 75 { 76 u8 flags = fscrypt_policy_flags(&ci->ci_policy); 77 78 memset(iv, 0, ci->ci_mode->ivsize); 79 80 if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { 81 WARN_ON_ONCE((u32)lblk_num != lblk_num); 82 lblk_num |= (u64)ci->ci_inode->i_ino << 32; 83 } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) { 84 memcpy(iv->nonce, ci->ci_nonce, FS_KEY_DERIVATION_NONCE_SIZE); 85 } 86 iv->lblk_num = cpu_to_le64(lblk_num); 87 } 88 89 /* Encrypt or decrypt a single filesystem block of file contents */ 90 int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, 91 u64 lblk_num, struct page *src_page, 92 struct page *dest_page, unsigned int len, 93 unsigned int offs, gfp_t gfp_flags) 94 { 95 union fscrypt_iv iv; 96 struct skcipher_request *req = NULL; 97 DECLARE_CRYPTO_WAIT(wait); 98 struct scatterlist dst, src; 99 struct fscrypt_info *ci = inode->i_crypt_info; 100 struct crypto_skcipher *tfm = ci->ci_ctfm; 101 int res = 0; 102 103 if (WARN_ON_ONCE(len <= 0)) 104 return -EINVAL; 105 if (WARN_ON_ONCE(len % FS_CRYPTO_BLOCK_SIZE != 0)) 106 return -EINVAL; 107 108 fscrypt_generate_iv(&iv, lblk_num, ci); 109 110 req = skcipher_request_alloc(tfm, gfp_flags); 111 if (!req) 112 return -ENOMEM; 113 114 skcipher_request_set_callback( 115 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 116 crypto_req_done, &wait); 117 118 sg_init_table(&dst, 1); 119 sg_set_page(&dst, dest_page, len, offs); 120 sg_init_table(&src, 1); 121 sg_set_page(&src, src_page, len, offs); 122 skcipher_request_set_crypt(req, &src, &dst, len, &iv); 123 if (rw == FS_DECRYPT) 124 res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait); 125 else 126 res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 127 skcipher_request_free(req); 128 if (res) { 129 fscrypt_err(inode, "%scryption failed for block %llu: %d", 130 (rw == FS_DECRYPT ? "De" : "En"), lblk_num, res); 131 return res; 132 } 133 return 0; 134 } 135 136 /** 137 * fscrypt_encrypt_pagecache_blocks() - Encrypt filesystem blocks from a pagecache page 138 * @page: The locked pagecache page containing the block(s) to encrypt 139 * @len: Total size of the block(s) to encrypt. Must be a nonzero 140 * multiple of the filesystem's block size. 141 * @offs: Byte offset within @page of the first block to encrypt. Must be 142 * a multiple of the filesystem's block size. 143 * @gfp_flags: Memory allocation flags 144 * 145 * A new bounce page is allocated, and the specified block(s) are encrypted into 146 * it. In the bounce page, the ciphertext block(s) will be located at the same 147 * offsets at which the plaintext block(s) were located in the source page; any 148 * other parts of the bounce page will be left uninitialized. However, normally 149 * blocksize == PAGE_SIZE and the whole page is encrypted at once. 150 * 151 * This is for use by the filesystem's ->writepages() method. 152 * 153 * Return: the new encrypted bounce page on success; an ERR_PTR() on failure 154 */ 155 struct page *fscrypt_encrypt_pagecache_blocks(struct page *page, 156 unsigned int len, 157 unsigned int offs, 158 gfp_t gfp_flags) 159 160 { 161 const struct inode *inode = page->mapping->host; 162 const unsigned int blockbits = inode->i_blkbits; 163 const unsigned int blocksize = 1 << blockbits; 164 struct page *ciphertext_page; 165 u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) + 166 (offs >> blockbits); 167 unsigned int i; 168 int err; 169 170 if (WARN_ON_ONCE(!PageLocked(page))) 171 return ERR_PTR(-EINVAL); 172 173 if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize))) 174 return ERR_PTR(-EINVAL); 175 176 ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags); 177 if (!ciphertext_page) 178 return ERR_PTR(-ENOMEM); 179 180 for (i = offs; i < offs + len; i += blocksize, lblk_num++) { 181 err = fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, 182 page, ciphertext_page, 183 blocksize, i, gfp_flags); 184 if (err) { 185 fscrypt_free_bounce_page(ciphertext_page); 186 return ERR_PTR(err); 187 } 188 } 189 SetPagePrivate(ciphertext_page); 190 set_page_private(ciphertext_page, (unsigned long)page); 191 return ciphertext_page; 192 } 193 EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); 194 195 /** 196 * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place 197 * @inode: The inode to which this block belongs 198 * @page: The page containing the block to encrypt 199 * @len: Size of block to encrypt. Doesn't need to be a multiple of the 200 * fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE. 201 * @offs: Byte offset within @page at which the block to encrypt begins 202 * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based 203 * number of the block within the file 204 * @gfp_flags: Memory allocation flags 205 * 206 * Encrypt a possibly-compressed filesystem block that is located in an 207 * arbitrary page, not necessarily in the original pagecache page. The @inode 208 * and @lblk_num must be specified, as they can't be determined from @page. 209 * 210 * Return: 0 on success; -errno on failure 211 */ 212 int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, 213 unsigned int len, unsigned int offs, 214 u64 lblk_num, gfp_t gfp_flags) 215 { 216 return fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, page, page, 217 len, offs, gfp_flags); 218 } 219 EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); 220 221 /** 222 * fscrypt_decrypt_pagecache_blocks() - Decrypt filesystem blocks in a pagecache page 223 * @page: The locked pagecache page containing the block(s) to decrypt 224 * @len: Total size of the block(s) to decrypt. Must be a nonzero 225 * multiple of the filesystem's block size. 226 * @offs: Byte offset within @page of the first block to decrypt. Must be 227 * a multiple of the filesystem's block size. 228 * 229 * The specified block(s) are decrypted in-place within the pagecache page, 230 * which must still be locked and not uptodate. Normally, blocksize == 231 * PAGE_SIZE and the whole page is decrypted at once. 232 * 233 * This is for use by the filesystem's ->readpages() method. 234 * 235 * Return: 0 on success; -errno on failure 236 */ 237 int fscrypt_decrypt_pagecache_blocks(struct page *page, unsigned int len, 238 unsigned int offs) 239 { 240 const struct inode *inode = page->mapping->host; 241 const unsigned int blockbits = inode->i_blkbits; 242 const unsigned int blocksize = 1 << blockbits; 243 u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) + 244 (offs >> blockbits); 245 unsigned int i; 246 int err; 247 248 if (WARN_ON_ONCE(!PageLocked(page))) 249 return -EINVAL; 250 251 if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize))) 252 return -EINVAL; 253 254 for (i = offs; i < offs + len; i += blocksize, lblk_num++) { 255 err = fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, 256 page, blocksize, i, GFP_NOFS); 257 if (err) 258 return err; 259 } 260 return 0; 261 } 262 EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); 263 264 /** 265 * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place 266 * @inode: The inode to which this block belongs 267 * @page: The page containing the block to decrypt 268 * @len: Size of block to decrypt. Doesn't need to be a multiple of the 269 * fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE. 270 * @offs: Byte offset within @page at which the block to decrypt begins 271 * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based 272 * number of the block within the file 273 * 274 * Decrypt a possibly-compressed filesystem block that is located in an 275 * arbitrary page, not necessarily in the original pagecache page. The @inode 276 * and @lblk_num must be specified, as they can't be determined from @page. 277 * 278 * Return: 0 on success; -errno on failure 279 */ 280 int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, 281 unsigned int len, unsigned int offs, 282 u64 lblk_num) 283 { 284 return fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, page, 285 len, offs, GFP_NOFS); 286 } 287 EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); 288 289 /* 290 * Validate dentries in encrypted directories to make sure we aren't potentially 291 * caching stale dentries after a key has been added. 292 */ 293 static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags) 294 { 295 struct dentry *dir; 296 int err; 297 int valid; 298 299 /* 300 * Plaintext names are always valid, since fscrypt doesn't support 301 * reverting to ciphertext names without evicting the directory's inode 302 * -- which implies eviction of the dentries in the directory. 303 */ 304 if (!(dentry->d_flags & DCACHE_ENCRYPTED_NAME)) 305 return 1; 306 307 /* 308 * Ciphertext name; valid if the directory's key is still unavailable. 309 * 310 * Although fscrypt forbids rename() on ciphertext names, we still must 311 * use dget_parent() here rather than use ->d_parent directly. That's 312 * because a corrupted fs image may contain directory hard links, which 313 * the VFS handles by moving the directory's dentry tree in the dcache 314 * each time ->lookup() finds the directory and it already has a dentry 315 * elsewhere. Thus ->d_parent can be changing, and we must safely grab 316 * a reference to some ->d_parent to prevent it from being freed. 317 */ 318 319 if (flags & LOOKUP_RCU) 320 return -ECHILD; 321 322 dir = dget_parent(dentry); 323 err = fscrypt_get_encryption_info(d_inode(dir)); 324 valid = !fscrypt_has_encryption_key(d_inode(dir)); 325 dput(dir); 326 327 if (err < 0) 328 return err; 329 330 return valid; 331 } 332 333 const struct dentry_operations fscrypt_d_ops = { 334 .d_revalidate = fscrypt_d_revalidate, 335 }; 336 337 /** 338 * fscrypt_initialize() - allocate major buffers for fs encryption. 339 * @cop_flags: fscrypt operations flags 340 * 341 * We only call this when we start accessing encrypted files, since it 342 * results in memory getting allocated that wouldn't otherwise be used. 343 * 344 * Return: 0 on success; -errno on failure 345 */ 346 int fscrypt_initialize(unsigned int cop_flags) 347 { 348 int err = 0; 349 350 /* No need to allocate a bounce page pool if this FS won't use it. */ 351 if (cop_flags & FS_CFLG_OWN_PAGES) 352 return 0; 353 354 mutex_lock(&fscrypt_init_mutex); 355 if (fscrypt_bounce_page_pool) 356 goto out_unlock; 357 358 err = -ENOMEM; 359 fscrypt_bounce_page_pool = 360 mempool_create_page_pool(num_prealloc_crypto_pages, 0); 361 if (!fscrypt_bounce_page_pool) 362 goto out_unlock; 363 364 err = 0; 365 out_unlock: 366 mutex_unlock(&fscrypt_init_mutex); 367 return err; 368 } 369 370 void fscrypt_msg(const struct inode *inode, const char *level, 371 const char *fmt, ...) 372 { 373 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 374 DEFAULT_RATELIMIT_BURST); 375 struct va_format vaf; 376 va_list args; 377 378 if (!__ratelimit(&rs)) 379 return; 380 381 va_start(args, fmt); 382 vaf.fmt = fmt; 383 vaf.va = &args; 384 if (inode) 385 printk("%sfscrypt (%s, inode %lu): %pV\n", 386 level, inode->i_sb->s_id, inode->i_ino, &vaf); 387 else 388 printk("%sfscrypt: %pV\n", level, &vaf); 389 va_end(args); 390 } 391 392 /** 393 * fscrypt_init() - Set up for fs encryption. 394 */ 395 static int __init fscrypt_init(void) 396 { 397 int err = -ENOMEM; 398 399 /* 400 * Use an unbound workqueue to allow bios to be decrypted in parallel 401 * even when they happen to complete on the same CPU. This sacrifices 402 * locality, but it's worthwhile since decryption is CPU-intensive. 403 * 404 * Also use a high-priority workqueue to prioritize decryption work, 405 * which blocks reads from completing, over regular application tasks. 406 */ 407 fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue", 408 WQ_UNBOUND | WQ_HIGHPRI, 409 num_online_cpus()); 410 if (!fscrypt_read_workqueue) 411 goto fail; 412 413 fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT); 414 if (!fscrypt_info_cachep) 415 goto fail_free_queue; 416 417 err = fscrypt_init_keyring(); 418 if (err) 419 goto fail_free_info; 420 421 return 0; 422 423 fail_free_info: 424 kmem_cache_destroy(fscrypt_info_cachep); 425 fail_free_queue: 426 destroy_workqueue(fscrypt_read_workqueue); 427 fail: 428 return err; 429 } 430 late_initcall(fscrypt_init) 431